package domainser import ( "context" "fmt" "math/rand" "net/http" "net/url" "stat/common/log" "stat/common/utils" "stat/models/domainmod" "stat/statg" "strings" "sync" "time" "go.mongodb.org/mongo-driver/bson" "github.com/ddliu/go-httpclient" ) const ( ApiKey = "a45d5d2c6c231f28f9bcccf4092eb246" ApiDomain = "https://api.boce.com" ApiCheckDuration = time.Second * 10 ApiCheckTimeout = time.Minute * 2 ApiCheckTryCount = 6 ) type NodeCache struct { List NodeList Last time.Time Lock sync.Mutex } type MutexResult struct { List domainmod.CheckResultList Lock sync.Mutex } var ( Cache = &NodeCache{ List: make(NodeList, 0), Last: time.Time{}, Lock: sync.Mutex{}, } ) func (c *NodeCache) NodeList() (NodeList, error) { ret := make(NodeList, 0) c.Lock.Lock() if len(c.List) > 0 && c.Last.After(time.Now().Add(-24*time.Hour)) { for _, node := range c.List { ret = append(ret, node) } c.Lock.Unlock() return ret, nil } c.Lock.Unlock() list, err := GetNodeList() if err != nil { return ret, err } if len(list) > 0 { c.Lock.Lock() c.List = list c.Last = time.Now() for _, node := range c.List { ret = append(ret, node) } c.Lock.Unlock() return ret, nil } else { return nil, fmt.Errorf("node list is empty") } } func GetNodeList() (NodeList, error) { addr := fmt.Sprintf("https://task.boce.com/v3/node/list?key=%s", ApiKey) res := &CommonResp{} tryCount := 0 for { err := Get(addr, res, url.Values{}) if err != nil { tryCount++ if tryCount > ApiCheckTryCount { return nil, err } time.Sleep(ApiCheckDuration) continue } break } if res.ErrorCode != 0 { return nil, fmt.Errorf("获取检测节点列表失败:%s", GetApiErrorMessage(res.ErrorCode)) } return *res.Data.List, nil } func (s *MutexResult) OnFinish(result *domainmod.CheckResult) { s.Lock.Lock() defer s.Lock.Unlock() s.List = append(s.List, result) } func (s *MutexResult) RealErrors() []error { ret := make([]error, 0) for _, item := range s.List { if item.CheckSucc && item.Error != nil { ret = append(ret, item.Error) } } return ret } func (s *MutexResult) AllSuccess() bool { for _, item := range s.List { if item.CheckSucc && item.Error != nil { return false } } return true } func (s *MutexResult) JumpErrors() []error { ret := make([]error, 0) for _, item := range s.List { if !item.CheckSucc && item.Error != nil { ret = append(ret, item.Error) } } return ret } func (s *MutexResult) Reports() string { var ret string for _, item := range s.List { if item.Error != nil { ret += "\n" + item.Error.Error() + "\n" } if len(item.NodeErrors) > 0 { ret += strings.Join(item.NodeErrors, "\n\n") } } return ret } func (s *MutexResult) CheckStatus() (passCount int, jumpCount int, errorCount int) { for _, item := range s.List { if item.CheckSucc { if item.Error != nil { errorCount++ } else { passCount++ } } else { jumpCount++ } } return } var ApiErrorMessage = map[int]string{ 0: "成功", 1: "鉴权失败", 2: "参数错误", 3: "波点不足或波点未配置", 4: "生成任务id失败或任务id失效", } func GetApiErrorMessage(code int) string { ret := ApiErrorMessage[code] if ret == "" { return "未知错误" } return ret } var ApiPathMap = map[domainmod.CheckType]ApiPath{ domainmod.CheckTypeWeb: { Create: "/v3/task/create/curl", Query: "/v3/task/curl/", Sleep: 5 * time.Minute, NeedCreate: true, }, domainmod.CheckTypeDNS: { Create: "/v3/task/create/dig", Query: "/v3/task/dig/", Sleep: 5 * time.Minute, NeedCreate: true, }, domainmod.CheckTypePing: { Create: "/v3/task/create/ping", Query: "/v3/task/ping/", Sleep: 5 * time.Minute, NeedCreate: true, }, domainmod.CheckTypeHijack: { Create: "/v3/task/create/hijack", Query: "/v3/task/hijack/", Sleep: 5 * time.Minute, NeedCreate: true, }, domainmod.CheckTypePollute: { Create: "", Query: "/v3/task/create/pollute", NeedCreate: false, }, domainmod.CheckTypeQQ: { Create: "", Query: "/v3/task/create/qq", NeedCreate: false, }, domainmod.CheckTypeWeChat: { Create: "", Query: "/v3/task/create/wechat", NeedCreate: false, }, domainmod.CheckTypeBlack: { Create: "", Query: "/v3/task/create/blacklist", NeedCreate: false, }, domainmod.CheckTypeGFW: { Create: "", Query: "/v3/task/create/wall", NeedCreate: false, }, } func init() { httpclient.Defaults(httpclient.Map{ "OPT_CONNECTTIMEOUT": 30 * time.Second, "OPT_TIMEOUT": 10 * time.Second, "Accept-Encoding": "gzip,deflate", }) } func GetChecker(item *domainmod.Domain, mode domainmod.CheckType, onFinish FnCheckFinish, nodes ...Node) Checker { nodeList := make(NodeList, 0) nodeList = append(nodeList, nodes...) switch mode { case domainmod.CheckTypeWeb: return NewWebChecker(item, mode, onFinish, nodeList) case domainmod.CheckTypeDNS: return NewWebChecker(item, mode, onFinish, nodeList) case domainmod.CheckTypePing: return NewWebChecker(item, mode, onFinish, nodeList) case domainmod.CheckTypeHijack: return NewHijackChecker(item, mode, onFinish, nodeList) case domainmod.CheckTypePollute: return NewPolluteChecker(item, mode, onFinish) case domainmod.CheckTypeQQ: return NewQQChecker(item, mode, onFinish) case domainmod.CheckTypeWeChat: return NewWeChatChecker(item, mode, onFinish) case domainmod.CheckTypeBlack: return NewBlackChecker(item, mode, onFinish) case domainmod.CheckTypeGFW: return NewGFWChecker(item, mode, onFinish) default: return nil } } type DomainChecker struct { Items domainmod.DomainList } func (c *DomainChecker) StartCheck() error { nodes, err := Cache.NodeList() if err != nil { return err } wg := utils.NewWaitGroup() for _, domain := range c.Items { _domain := domain wg.Add(func() { CheckDomain(_domain, nodes) }) } wg.RunWaits() return nil } func CheckDomain(domain *domainmod.Domain, nodes NodeList) { ctx := context.Background() begin := time.Now() status := domainmod.CheckStatusChecking doc := &domainmod.DomainDoc{ Status: &status, CheckTime: &begin, UpdatedAt: &begin, } if err := domainmod.UpdateOneDomain(ctx, bson.M{"_id": domain.ID}, bson.M{"$set": doc}); err != nil { log.Warn("CheckDomain failed", log.E(err)) return } rets := MutexResult{ List: make(domainmod.CheckResultList, 0), Lock: sync.Mutex{}, } wg := utils.NewWaitGroup() for _, check := range domain.Checks { _check := check wg.Add(func() { if checker := GetChecker(domain, _check, rets.OnFinish, nodes...); checker != nil { time.Sleep(time.Duration(rand.Intn(10)) * time.Second) // 防并发访问 checker.Check() } }) } wg.RunWaits() end := time.Now() doc = &domainmod.DomainDoc{ CheckFinish: &end, UpdatedAt: &end, } passCount, jumpCount, errCount := rets.CheckStatus() doc.Results = &rets.List if errCount > 0 { report := rets.Reports() doc.Report = &report status0 := domainmod.CheckStatusFailed //doc.Status = utils.PValue(domainmod.CheckStatusFailed) if passCount > 0 { //doc.Status = utils.PValue(domainmod.CheckStatusPartPass) status0 = domainmod.CheckStatusPartPass } doc.Status = &status0 } else { if passCount > 0 { if jumpCount > 0 { //doc.Status = utils.PValue(domainmod.CheckStatusPartPass) //doc.Report = utils.PValue(rets.Reports()) status0 := domainmod.CheckStatusPartPass report := rets.Reports() doc.Status = &status0 doc.Report = &report } else { //doc.Status = utils.PValue(domainmod.CheckStatusSucc) //doc.Report = utils.PValue("完全通过") status0 := domainmod.CheckStatusSucc report := "完全通过" doc.Status = &status0 doc.Report = &report } } else { //doc.Status = utils.PValue(domainmod.CheckStatusNone) //doc.Report = utils.PValue(rets.Reports()) status0 := domainmod.CheckStatusNone report := rets.Reports() doc.Status = &status0 doc.Report = &report } } if err := domainmod.UpdateOneDomain(ctx, bson.M{"_id": domain.ID}, bson.M{"$set": doc}); err != nil { log.Warn("CheckDomain failed", log.E(err)) return } if errCount > 0 { text := fmt.Sprintf("域名检测:【%s】\n", domain.Name) text += fmt.Sprintf("开始时间:【%s】\n", begin.Format("2006-01-02T15:04:05")) text += fmt.Sprintf("结束时间:【%s】\n", end.Format("2006-01-02T15:04:05")) text += fmt.Sprintf("检测用时:【%s】\n", end.Sub(begin)) text += fmt.Sprintf("检测结果:【%s】\n", *doc.Status) for _, result := range rets.List { if result.CheckSucc { if result.Error == nil { text += fmt.Sprintf("%s: %s\n", result.CheckType.Name(), "\U0001F7E2 通过") } else { if result.NodeCount == 0 { text += fmt.Sprintf("%s: %s\n", result.CheckType.Name(), "🔴 未通过") } else { if result.SuccRate < result.MinRate { if result.SuccRate < 25 { text += fmt.Sprintf("%s: %s\n", result.CheckType.Name(), "🔴 未通过") } else { text += fmt.Sprintf("%s: %s\n", result.CheckType.Name(), "\U0001F7E0 通过") } } else { text += fmt.Sprintf("%s: %s\n", result.CheckType.Name(), "\U0001F7E2 通过") } } } } else { text += fmt.Sprintf("%s: %s\n", result.CheckType.Name(), "🕓 超时") } } if domain.Redirected { text += fmt.Sprintf("域名跳转:【已做301跳转✅】\n") text += fmt.Sprintf("跳转地址:【==>%s】\n", domain.RedirectDomain) } else { text += fmt.Sprintf("域名跳转:【未做301跳转🈚】\n") } statg.Bot.SendDomainMessage(text) } } type WebChecker struct { Item *domainmod.Domain Mode domainmod.CheckType Finish FnCheckFinish Nodes NodeList } func NewWebChecker(item *domainmod.Domain, mode domainmod.CheckType, onFinish FnCheckFinish, nodes NodeList) *WebChecker { return &WebChecker{ Item: item, Mode: mode, Finish: onFinish, Nodes: nodes, } } func (s *WebChecker) Check() error { ret := &domainmod.CheckResult{ Domain: s.Item.Name, CheckType: s.Mode, BeginTime: time.Now(), MinRate: s.Item.MinRate, } defer s.Finish(ret) api, ok := ApiPathMap[s.Mode] if !ok { ret.EndTime = time.Now() ret.Error = fmt.Errorf("域名%s的检测类型%d不支持", s.Item.Name, s.Mode) return ret.Error } addr, err := api.GetCreateUrl(url.Values{ "node_ids": []string{s.Nodes.SIdList(s.Item.MaxNodes)}, "host": []string{s.Item.Name}, }) if err != nil { ret.EndTime = time.Now() ret.Error = err return ret.Error } tryCount := 0 res := &CommonResp{} for { err = Get(addr, res, url.Values{}) if err != nil { tryCount++ if tryCount > ApiCheckTryCount { ret.EndTime = time.Now() ret.Error = fmt.Errorf("创建【%s】的【%s】检测任务超时:%s", s.Item.Name, s.Mode.Name(), err.Error()) return ret.Error } time.Sleep(ApiCheckDuration) continue } break } if res.ErrorCode != 0 { ret.EndTime = time.Now() ret.Error = fmt.Errorf("创建【%s】的【%s】检测任务失败:%s", s.Item.Name, s.Mode.Name(), GetApiErrorMessage(res.ErrorCode)) return ret.Error } // 获取查询ID if res.Data.Id == nil || *res.Data.Id == "" { ret.EndTime = time.Now() ret.Error = fmt.Errorf("创建【%s】的【%s】检测任务失败:任务ID未空", s.Item.Name, s.Mode.Name()) return ret.Error } TaskId := *res.Data.Id // 去查询结果 tryCount = 0 addr, err = api.GetQueryUrlWithTaskId(TaskId, url.Values{}) end := time.Now().Add(ApiCheckTimeout) for { time.Sleep(ApiCheckDuration) webResp := &WebResp{} err = Get(addr, webResp, url.Values{}) if err != nil { tryCount++ if tryCount > ApiCheckTryCount { ret.EndTime = time.Now() ret.Error = fmt.Errorf("查询【%s】的【%s】检测结果超时:%s", s.Item.Name, s.Mode.Name(), err.Error()) return ret.Error } continue } ret.CheckSucc = webResp.Done if webResp.Done { for _, node := range webResp.List { if node.ErrorCode != 0 { ret.FailedCount++ ret.NodeErrors = append(ret.NodeErrors, node.ErrorString()) } else { ret.NodeCount++ if node.HttpCode == http.StatusOK { ret.SuccCount++ } else { ret.NodeErrors = append(ret.NodeErrors, node.ErrorString()) } } } if ret.NodeCount > 0 { ret.SuccRate = float64(ret.SuccCount*100) / float64(ret.NodeCount) if ret.SuccRate < ret.MinRate { ret.Error = fmt.Errorf("节点通过率%.1f%%<%.1f%%,未通过!", ret.SuccRate, ret.MinRate) } } break } else { if end.Before(time.Now()) { ret.Error = fmt.Errorf("查询【%s】的【%s】检测超时", s.Item.Name, s.Mode.Name()) break } } } ret.EndTime = time.Now() return ret.Error } type HijackChecker struct { Item *domainmod.Domain Mode domainmod.CheckType Finish FnCheckFinish Nodes NodeList } func NewHijackChecker(item *domainmod.Domain, mode domainmod.CheckType, onFinish FnCheckFinish, nodes NodeList) *HijackChecker { return &HijackChecker{ Item: item, Mode: mode, Finish: onFinish, Nodes: nodes, } } func (s *HijackChecker) Check() error { ret := &domainmod.CheckResult{ Domain: s.Item.Name, CheckType: s.Mode, BeginTime: time.Now(), MinRate: s.Item.MinRate, } defer s.Finish(ret) api, ok := ApiPathMap[s.Mode] if !ok { ret.EndTime = time.Now() ret.Error = fmt.Errorf("域名%s的检测类型%d不支持", s.Item.Name, s.Mode) return ret.Error } addr, err := api.GetCreateUrl(url.Values{ "node_ids": []string{s.Nodes.SIdList(s.Item.MaxNodes)}, "host": []string{s.Item.Name}, }) if err != nil { ret.EndTime = time.Now() ret.Error = err return ret.Error } tryCount := 0 res := &CommonResp{} for { err = Get(addr, res, url.Values{}) if err != nil { tryCount++ if tryCount > ApiCheckTryCount { ret.EndTime = time.Now() ret.Error = fmt.Errorf("创建【%s】的【%s】检测任务超时:%s", s.Item.Name, s.Mode.Name(), err.Error()) return ret.Error } time.Sleep(ApiCheckDuration) continue } break } if res.ErrorCode != 0 { ret.EndTime = time.Now() ret.Error = fmt.Errorf("创建【%s】的【%s】检测任务失败:%s", s.Item.Name, s.Mode.Name(), GetApiErrorMessage(res.ErrorCode)) return ret.Error } // 获取查询ID if res.Data.Id == nil || *res.Data.Id == "" { ret.EndTime = time.Now() ret.Error = fmt.Errorf("创建【%s】的【%s】检测任务失败:任务ID未空", s.Item.Name, s.Mode.Name()) return ret.Error } TaskId := *res.Data.Id // 去查询结果 tryCount = 0 addr, err = api.GetQueryUrlWithTaskId(TaskId, url.Values{}) end := time.Now().Add(ApiCheckTimeout) for { time.Sleep(ApiCheckDuration) webResp := &HijackResp{} err = Get(addr, webResp, url.Values{}) if err != nil { tryCount++ if tryCount > ApiCheckTryCount { ret.EndTime = time.Now() ret.Error = fmt.Errorf("查询【%s】的【%s】接口错误【%d】次,超过最大重试次数:%s", s.Item.Name, s.Mode.Name(), tryCount, err.Error()) return ret.Error } continue } ret.CheckSucc = webResp.Done if webResp.Done { for _, node := range webResp.List { ret.NodeCount++ if node.Hijack { ret.FailedCount++ ret.NodeErrors = append(ret.NodeErrors, node.ErrorString()) } else { ret.SuccCount++ } } if ret.NodeCount > 0 { ret.SuccRate = float64(ret.SuccCount*100) / float64(ret.NodeCount) if ret.SuccRate < ret.MinRate { ret.Error = fmt.Errorf("节点通过率%.1f%%<%.1f%%,未通过!", ret.SuccRate, ret.MinRate) } } break } else { if end.Before(time.Now()) { ret.Error = fmt.Errorf("查询【%s】的【%s】检测超时", s.Item.Name, s.Mode.Name()) break } } } ret.EndTime = time.Now() return ret.Error } type FuncCheckResult func(ret *domainmod.CheckResult, data map[string]int) func QueryCheck(item *domainmod.Domain, mode domainmod.CheckType, onFinish FnCheckFinish, onCheck FuncCheckResult) error { ret := &domainmod.CheckResult{ Domain: item.Name, CheckType: mode, BeginTime: time.Now(), MinRate: item.MinRate, } defer onFinish(ret) api, ok := ApiPathMap[mode] if !ok { ret.EndTime = time.Now() ret.Error = fmt.Errorf("查询接口地址【%s】的【%s】出错", item.Name, mode.Name()) return ret.Error } addr, err := api.GetQueryUrl(url.Values{"host": []string{item.Name}}) polluteResp := &PolluteResp{} tryCount := 0 for { tryCount++ err = Get(addr, polluteResp, url.Values{}) if err != nil { if tryCount > ApiCheckTryCount { ret.EndTime = time.Now() ret.Error = fmt.Errorf("查询【%s】的【%s】接口错误【%d】次,超过最大重试次数:%s", item.Name, mode.Name(), tryCount, err.Error()) return ret.Error } time.Sleep(ApiCheckDuration) continue } break } if polluteResp.ErrorCode != 0 { ret.EndTime = time.Now() ret.Error = fmt.Errorf("查询【%s】的【%s】返回值不是成功:%s", item.Name, mode.Name(), GetApiErrorMessage(polluteResp.ErrorCode)) return ret.Error } ret.CheckSucc = true onCheck(ret, polluteResp.Data) ret.EndTime = time.Now() return ret.Error } type PolluteChecker struct { Item *domainmod.Domain Mode domainmod.CheckType Finish FnCheckFinish } func NewPolluteChecker(item *domainmod.Domain, mode domainmod.CheckType, onFinish FnCheckFinish) *PolluteChecker { return &PolluteChecker{ Item: item, Mode: mode, Finish: onFinish, } } func (s *PolluteChecker) Check() error { return QueryCheck(s.Item, s.Mode, s.Finish, func(ret *domainmod.CheckResult, data map[string]int) { if code, ok := data[s.Item.Name]; ok { //(-1 未注册,0 正常, 1 污染) if ok { if code == 0 { ret.SuccRate = 100.0 } else { if code == 1 { ret.Error = fmt.Errorf("域名被污染") } else if code == -1 { ret.Error = fmt.Errorf("域名未注册") } else { ret.Error = fmt.Errorf("未知错误码%d", code) } } } else { ret.Error = fmt.Errorf("返回值中未检测到域名") } } }) } type QQChecker struct { Item *domainmod.Domain Mode domainmod.CheckType Finish FnCheckFinish } func NewQQChecker(item *domainmod.Domain, mode domainmod.CheckType, onFinish FnCheckFinish) *QQChecker { return &QQChecker{ Item: item, Mode: mode, Finish: onFinish, } } func (s *QQChecker) Check() error { return QueryCheck(s.Item, s.Mode, s.Finish, func(ret *domainmod.CheckResult, data map[string]int) { if code, ok := data[s.Item.Name]; ok { //1正常, 2拦截, 3-检测失败 if ok { if code == 1 { ret.CheckSucc = true ret.SuccRate = 100.0 } else { if code == 2 { ret.CheckSucc = true ret.Error = fmt.Errorf("已被拦截") } else if code == 3 { ret.Error = fmt.Errorf("检测失败") } else { ret.Error = fmt.Errorf("检测值【%d】不知道意义", code) } } } else { ret.Error = fmt.Errorf("返回值中未检测到域名") } } }) } type WeChatChecker struct { Item *domainmod.Domain Mode domainmod.CheckType Finish FnCheckFinish } func NewWeChatChecker(item *domainmod.Domain, mode domainmod.CheckType, onFinish FnCheckFinish) *WeChatChecker { return &WeChatChecker{ Item: item, Mode: mode, Finish: onFinish, } } func (s *WeChatChecker) Check() error { return QueryCheck(s.Item, s.Mode, s.Finish, func(ret *domainmod.CheckResult, data map[string]int) { if code, ok := data[s.Item.Name]; ok { //1正常, 2拦截(域名被封), 3-检测失败,4-拦截(非微信官方链接,需跳转访问) if ok { if code == 1 || code == 4 { ret.CheckSucc = true ret.SuccRate = 100.0 } else { if code == 2 { ret.CheckSucc = true ret.Error = fmt.Errorf("已被拦截") } else if code == 3 { ret.Error = fmt.Errorf("检测失败") } else { ret.Error = fmt.Errorf("检测值【%d】不知道意义", code) } } } else { ret.Error = fmt.Errorf("返回值中未检测到域名") } } }) } type BlackChecker struct { Item *domainmod.Domain Mode domainmod.CheckType Finish FnCheckFinish } func NewBlackChecker(item *domainmod.Domain, mode domainmod.CheckType, onFinish FnCheckFinish) *BlackChecker { return &BlackChecker{ Item: item, Mode: mode, Finish: onFinish, } } func (s *BlackChecker) Check() error { return QueryCheck(s.Item, s.Mode, s.Finish, func(ret *domainmod.CheckResult, data map[string]int) { if code, ok := data[s.Item.Name]; ok { //1-正常, 2-在黑名单 if ok { if code == 1 { ret.CheckSucc = true ret.SuccRate = 100.0 } else { if code == 2 { ret.CheckSucc = true ret.Error = fmt.Errorf("在黑名单") } else { ret.Error = fmt.Errorf("检测值【%d】不知道意义", code) } } } else { ret.Error = fmt.Errorf("返回值中未检测到域名") } } }) } type GFWChecker struct { Item *domainmod.Domain Mode domainmod.CheckType Finish FnCheckFinish } func NewGFWChecker(item *domainmod.Domain, mode domainmod.CheckType, onFinish FnCheckFinish) *GFWChecker { return &GFWChecker{ Item: item, Mode: mode, Finish: onFinish, } } func (s *GFWChecker) Check() error { return QueryCheck(s.Item, s.Mode, s.Finish, func(ret *domainmod.CheckResult, data map[string]int) { if code, ok := data[s.Item.Name]; ok { //0 正常, 1 被墙, 2 疑似被墙, 3 无资源请重试,4 域名格式错误 if ok { if code == 0 { // 正常 ret.CheckSucc = true ret.SuccRate = 100.0 } else { if code == 1 { ret.CheckSucc = true ret.Error = fmt.Errorf("域名被墙") } else if code == 2 { ret.CheckSucc = true ret.Error = fmt.Errorf("疑似被墙") } else if code == 3 { ret.Error = fmt.Errorf("无资源请重试") } else if code == 4 { ret.Error = fmt.Errorf("域名格式错误") } else { ret.Error = fmt.Errorf("检测值【%d】不知道意义", code) } } } else { ret.Error = fmt.Errorf("返回值中未检测到域名") } } }) }