260 lines
8.9 KiB
Go
260 lines
8.9 KiB
Go
package pollute
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/xxjwxc/gowp/workpool"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"net/http"
|
|
"net/url"
|
|
"stat/common/log"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
//http://www.ce8.com/check
|
|
type ce8PolluteServerConfig struct {
|
|
DotNetApi string `json:"dotNetApi"`
|
|
}
|
|
type ce8PollutionFullReq struct {
|
|
URL string `json:"url"`
|
|
}
|
|
type ce8PollutionFullResp struct {
|
|
Code int `json:"code"`
|
|
State bool `json:"state"`
|
|
Runtime interface{} `json:"runtime"`
|
|
Data struct {
|
|
URL string `json:"url"`
|
|
Pollution bool `json:"pollution"`
|
|
PollutionNodeCount int `json:"pollution_node_count"`
|
|
NodeList []struct {
|
|
PollutionResult struct {
|
|
IP string `json:"ip"`
|
|
Location string `json:"location"`
|
|
Pollution bool `json:"pollution"`
|
|
IsTimeout bool `json:"is_timeout"`
|
|
} `json:"pollution_result"`
|
|
} `json:"node_list"`
|
|
} `json:"data"`
|
|
}
|
|
|
|
func ce8PolluteCheck(domains []CheckReq) ([]CheckResult, error) {
|
|
if len(domains) == 0 {
|
|
return nil, nil
|
|
}
|
|
doc, err := ce8GetCheckDoc()
|
|
if err != nil {
|
|
log.Warn("ce8PolluteCheck ce8GetCheckDoc err", log.Any("domain", domains), log.E(err))
|
|
return nil, err
|
|
}
|
|
sc, err := ce8GetPolluteServerConfig(doc)
|
|
if err != nil {
|
|
log.Warn("ce8PolluteCheck ce8GetPolluteServerConfig err", log.Any("domain", domains), log.E(err))
|
|
return nil, err
|
|
}
|
|
csrfToken := ce8GetCSRFToken(doc)
|
|
if csrfToken == "" {
|
|
log.Warn("ce8PolluteCheck ce8GetCSRFToken empty", log.Any("domain", domains))
|
|
}
|
|
log.Info("ce8PolluteCheck ce8PollutionFulls start", log.Any("domainsLen", len(domains)))
|
|
checkResult, err := ce8PollutionFulls(domains, *sc, csrfToken)
|
|
if err != nil {
|
|
log.Info("ce8PolluteCheck ce8PollutionFulls Finish Err", log.Any("domain", domains), log.E(err))
|
|
return nil, err
|
|
}
|
|
log.Info("ce8PolluteCheck ce8PollutionFulls Finish OK", log.Any("checkResultLen", len(checkResult)))
|
|
return checkResult, err
|
|
}
|
|
|
|
func ce8GetCheckDoc() (*goquery.Document, error) {
|
|
const ce8CheckURL = "https://ce8.com/define"
|
|
req, err := http.NewRequest("GET", ce8CheckURL, nil)
|
|
if err != nil {
|
|
log.Warn("ce8GetCheckDoc newRequest", log.Any("url", ce8CheckURL), log.E(err))
|
|
return nil, err
|
|
}
|
|
req.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
|
req.Header.Add("Accept-Language", " en-US,en;q=0.9")
|
|
req.Header.Add("Accept-Encoding", "gzip")
|
|
req.Header.Add("Cache-Control", "no-cache")
|
|
req.Header.Add("Connection", "keep-alive")
|
|
req.Header.Add("Host", "www.ce8.com")
|
|
req.Header.Add("Pragma", "no-cache")
|
|
req.Header.Add("sec-ch-ua", "\"Chromium\";v=\"88\", \"Google Chrome\";v=\"88\", \";Not A Brand\";v=\"99\"")
|
|
req.Header.Add("sec-ch-ua-mobile", "?0")
|
|
req.Header.Add("Sec-Fetch-Dest", "document")
|
|
req.Header.Add("Sec-Fetch-Mode", "navigate")
|
|
req.Header.Add("Sec-Fetch-Site", "none")
|
|
req.Header.Add("Sec-Fetch-User", "?1")
|
|
req.Header.Add("Upgrade-Insecure-Requests", "1")
|
|
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36")
|
|
res, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
log.Warn("ce8GetCheckDoc Client Do", log.Any("url", ce8CheckURL), log.E(err))
|
|
return nil, err
|
|
}
|
|
defer func() { _ = res.Body.Close() }()
|
|
if res.StatusCode != http.StatusOK {
|
|
log.Warn("ce8GetCheckDoc err http statusCode", log.Any("statusCode", res.StatusCode))
|
|
return nil, errors.New(fmt.Sprintf("StatusCode:%d", res.StatusCode))
|
|
}
|
|
bodyReader := res.Body
|
|
switch res.Header.Get("Content-Encoding") {
|
|
case "gzip":
|
|
bodyReader, err = gzip.NewReader(res.Body)
|
|
if err != nil {
|
|
log.Warn("ce8GetCheckDoc gzip newReader", log.E(err))
|
|
return nil, err
|
|
}
|
|
defer func() { _ = bodyReader.Close() }()
|
|
default:
|
|
}
|
|
doc, err := goquery.NewDocumentFromReader(bodyReader)
|
|
if err != nil {
|
|
log.Warn("ce8GetCheckDoc NewDocumentFromReader err", log.Any("url", ce8CheckURL), log.E(err))
|
|
return nil, err
|
|
}
|
|
return doc, nil
|
|
}
|
|
|
|
func ce8GetPolluteServerConfig(doc *goquery.Document) (*ce8PolluteServerConfig, error) {
|
|
script := doc.Find("script").Text()
|
|
scriptLines := strings.Split(script, "\n")
|
|
var serverConfigLine = ""
|
|
for _, line := range scriptLines {
|
|
if strings.Contains(line, "serverConfig") && strings.Contains(line, "dotNetApi") {
|
|
serverConfigLine = line
|
|
break
|
|
}
|
|
}
|
|
if serverConfigLine == "" {
|
|
log.Warn("ce8GetPolluteServerConfig Get Line Fail")
|
|
return nil, errors.New("not found serverConfig")
|
|
}
|
|
start := strings.Index(serverConfigLine, "{")
|
|
end := strings.LastIndex(serverConfigLine, "}")
|
|
if start == -1 || end == -1 {
|
|
log.Warn("ce8GetPolluteServerConfig err Line Content", log.Any("line", serverConfigLine))
|
|
return nil, errors.New("parseServerConfig error")
|
|
}
|
|
jsonString := serverConfigLine[start : end+1]
|
|
var serverConfig ce8PolluteServerConfig
|
|
err := json.Unmarshal([]byte(jsonString), &serverConfig)
|
|
if err != nil {
|
|
log.Warn("ce8GetPolluteServerConfig unmarshal ServerConfig err", log.E(err))
|
|
return nil, err
|
|
}
|
|
if serverConfig.DotNetApi == "" {
|
|
log.Warn("ce8GetPolluteServerConfig dotNetApi empty", log.Any("jsonString", jsonString))
|
|
return nil, errors.New("serverConfig dotNetApi empty")
|
|
}
|
|
return &serverConfig, nil
|
|
}
|
|
|
|
func ce8GetCSRFToken(doc *goquery.Document) string {
|
|
var csrfToken = ""
|
|
doc.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
|
|
if name, _ := s.Attr("name"); name == "csrf-token" {
|
|
csrfToken, _ = s.Attr("content")
|
|
return false
|
|
}
|
|
return true
|
|
})
|
|
return csrfToken
|
|
}
|
|
|
|
func ce8PollutionFulls(reqs []CheckReq, sc ce8PolluteServerConfig, csrfToken string) ([]CheckResult, error) {
|
|
ch := make(chan CheckResult, len(reqs))
|
|
wp := workpool.New(15)
|
|
reqs = reqs[:] //copy
|
|
//在服务器上经常出现超时，这里先做下随机
|
|
rand.Shuffle(len(reqs), func(i, j int) { reqs[i], reqs[j] = reqs[j], reqs[i] })
|
|
for i := range reqs {
|
|
req := reqs[i]
|
|
wp.Do(func() error {
|
|
domain := req.Domain
|
|
r := CheckResult{Domain: domain, Payload: req.Payload, Err: errors.New("exception occurred")}
|
|
polled, err := ce8PollutionFull(domain, sc, csrfToken)
|
|
r.Polled = polled
|
|
r.Err = err
|
|
ch <- r
|
|
return nil
|
|
})
|
|
}
|
|
err := wp.Wait()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var result []CheckResult
|
|
count := len(ch)
|
|
for i := 0; i < count; i++ {
|
|
result = append(result, <-ch)
|
|
}
|
|
close(ch)
|
|
ch = nil
|
|
|
|
return result, nil
|
|
}
|
|
|
|
func ce8PollutionFull(domain string, sc ce8PolluteServerConfig, csrfToken string) (bool, error) {
|
|
u, err := url.Parse(sc.DotNetApi)
|
|
if err != nil {
|
|
log.Warn("ce8PollutionFull parseAPI", log.Any("DotNetApi", sc.DotNetApi), log.E(err))
|
|
return false, err
|
|
}
|
|
u.Path = "/api/define/pollution_full"
|
|
reqBody, _ := json.Marshal(&ce8PollutionFullReq{URL: domain})
|
|
checkURL := u.String()
|
|
req, err := http.NewRequest("POST", checkURL, bytes.NewReader(reqBody))
|
|
if err != nil {
|
|
log.Warn("ce8PollutionFull newRequest", log.Any("url", checkURL), log.E(err))
|
|
return false, err
|
|
}
|
|
req.Header.Add("Connection", "keep-alive")
|
|
req.Header.Add("Pragma", "no-cache")
|
|
req.Header.Add("Cache-Control", "no-cache")
|
|
req.Header.Add("Accept", "application/json, text/javascript, */*; q=0.01")
|
|
if csrfToken != "" {
|
|
req.Header.Add("X-CSRF-TOKEN", csrfToken)
|
|
}
|
|
req.Header.Add("User-Agent", "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36")
|
|
req.Header.Add("Content-Type", "application/json; charset=UTF-8")
|
|
req.Header.Add("Origin", "http://www.ce8.com")
|
|
req.Header.Add("Referer", "http://www.ce8.com/")
|
|
req.Header.Add("Accept-Language", "en-US,en;q=0.9")
|
|
client := http.Client{Timeout: 150 * time.Second}
|
|
res, err := client.Do(req)
|
|
if err != nil {
|
|
log.Warn("ce8PollutionFull Client Do", log.Any("checkURL", checkURL), log.Any("domain", domain), log.E(err))
|
|
return false, err
|
|
}
|
|
defer func() { _ = res.Body.Close() }()
|
|
if res.StatusCode != http.StatusOK {
|
|
log.Warn("ce8PollutionFull Resp not OK", log.Any("checkURL", checkURL), log.Any("domain", domain), log.Any("statusCode", res.StatusCode))
|
|
return false, errors.New(fmt.Sprintf("StatusCode:%d", res.StatusCode))
|
|
}
|
|
body, err := ioutil.ReadAll(res.Body)
|
|
if err != nil {
|
|
log.Warn("ce8PollutionFull Resp ReadAll", log.Any("checkURL", checkURL), log.Any("domain", domain), log.E(err))
|
|
return false, err
|
|
}
|
|
var fullResp ce8PollutionFullResp
|
|
err = json.Unmarshal(body, &fullResp)
|
|
if err != nil {
|
|
log.Warn("ce8PollutionFull Unmarshal", log.Any("checkURL", checkURL), log.Any("domain", domain), log.Any("body", body), log.E(err))
|
|
return false, err
|
|
}
|
|
if fullResp.Code == 0 {
|
|
log.Info("ce8PollutionFull Code OK", log.Any("checkURL", checkURL), log.Any("domain", domain), log.Any("pollution", fullResp.Data.Pollution))
|
|
return fullResp.Data.Pollution, nil
|
|
}
|
|
log.Warn("ce8PollutionFull Code", log.Any("checkURL", checkURL), log.Any("domain", domain), log.Any("code", fullResp.Code))
|
|
return false, errors.New(fmt.Sprintf("respcode expect 0,got:%d", fullResp.Code))
|
|
}
|