// Package indexnow 实现 IndexNow 协议提交,推送到官方入口后由 Bing / Yandex 等共享数据。 // 调用方是 web 进程的 seopush,在文章发布 / 更新 / 删除时推送对应页面地址。 package indexnow import ( "bytes" "encoding/json" "fmt" "net/http" "net/url" "strings" "time" "videoweb/pub/common/log" ) // DefaultKey IndexNow 默认密钥,与站点根目录下的 {key}.txt 验证文件对应 const DefaultKey = "6d41643810d540ce924edb982c3d3171" // DefaultSearchEngine 官方入口,会自动分发给 Bing、Yandex 等搜索引擎 const DefaultSearchEngine = "https://api.indexnow.org" // IndexNowRequest 提交请求结构体 type IndexNowRequest struct { Host string `json:"host"` Key string `json:"key"` KeyLocation string `json:"keyLocation,omitempty"` URLList []string `json:"urlList"` } // IndexNowConfig 配置结构体 type IndexNowConfig struct { Key string Host string KeyLocation string SearchEngines []string RetryCount int RetryDelay time.Duration // BatchInterval 多批次之间的间隔,避免触发频率限制;单批提交不生效 BatchInterval time.Duration } // BuildConfig 依据站点地址推导 IndexNow 配置 func BuildConfig(webAddr string) (*IndexNowConfig, error) { host, keyLocation, err := hostFromWebAddr(webAddr, DefaultKey) if err != nil { return nil, err } return &IndexNowConfig{ Key: DefaultKey, Host: host, KeyLocation: keyLocation, SearchEngines: []string{DefaultSearchEngine}, RetryCount: 3, RetryDelay: 2 * time.Second, BatchInterval: 60 * time.Second, }, nil } func hostFromWebAddr(webAddr, key string) (host, keyLocation string, err error) { normalized := webAddr if !strings.HasPrefix(normalized, "http://") && !strings.HasPrefix(normalized, "https://") { normalized = "https://" + normalized } u, parseErr := url.Parse(normalized) if parseErr != nil { return "", "", parseErr } host = u.Hostname() if host == "" { return "", "", fmt.Errorf("invalid webAddr: %s", webAddr) } base := strings.TrimSuffix(normalized, "/") keyLocation = fmt.Sprintf("%s/%s.txt", base, key) return host, keyLocation, nil } // BatchIndexNowSubmitter 批量提交器 type BatchIndexNowSubmitter struct { config *IndexNowConfig client *http.Client } // NewBatchIndexNowSubmitter 创建新的批量提交器 func NewBatchIndexNowSubmitter(config *IndexNowConfig) *BatchIndexNowSubmitter { if config.RetryCount == 0 { config.RetryCount = 3 } if config.RetryDelay == 0 { config.RetryDelay = 1 * time.Second } if config.BatchInterval == 0 { config.BatchInterval = 60 * time.Second } return &BatchIndexNowSubmitter{ config: config, client: &http.Client{ Timeout: 30 * time.Second, }, } } // SubmitBatch 批量提交URL,返回成功提交的 URL 列表 func (s *BatchIndexNowSubmitter) SubmitBatch(urls []string) *BatchResult { result := &BatchResult{ TotalURLs: len(urls), SuccessCount: 0, FailCount: 0, Errors: make([]string, 0), SuccessURLs: make([]string, 0), StatusCodes: make([]int, 0), } if len(urls) == 0 { return result } // 分批处理,每批最多100个URL(避免单次请求过大) const batchSize = 100 batches := s.splitIntoBatches(urls, batchSize) for batchIndex, batchURLs := range batches { codes, err := s.submitWithRetry(batchURLs) result.StatusCodes = append(result.StatusCodes, codes...) if err != nil { result.FailCount += len(batchURLs) result.Errors = append(result.Errors, fmt.Sprintf("批次 %d 提交失败: %v", batchIndex+1, err)) } else { result.SuccessCount += len(batchURLs) result.SuccessURLs = append(result.SuccessURLs, batchURLs...) } if batchIndex < len(batches)-1 { time.Sleep(s.config.BatchInterval) } } return result } // submitWithRetry 带重试的提交,返回各引擎 HTTP 状态码 func (s *BatchIndexNowSubmitter) submitWithRetry(urls []string) (statusCodes []int, err error) { var lastError error var lastCodes []int for attempt := 1; attempt <= s.config.RetryCount; attempt++ { codes, submitErr := s.submitToAllEngines(urls) lastCodes = codes if submitErr != nil { lastError = submitErr if attempt < s.config.RetryCount { log.Warn("indexnow: submit retry", log.Any("attempt", attempt), log.Any("delay", s.config.RetryDelay), log.Any("statusCodes", codes), log.E(submitErr)) time.Sleep(s.config.RetryDelay) continue } } else { return codes, nil } } return lastCodes, fmt.Errorf("经过 %d 次重试后仍然失败: %v", s.config.RetryCount, lastError) } // submitToAllEngines 向所有搜索引擎提交,返回各引擎 HTTP 状态码 func (s *BatchIndexNowSubmitter) submitToAllEngines(urls []string) (statusCodes []int, err error) { requestBody := IndexNowRequest{ Host: s.config.Host, Key: s.config.Key, KeyLocation: s.config.KeyLocation, URLList: urls, } jsonData, err := json.Marshal(requestBody) if err != nil { return nil, fmt.Errorf("JSON编码失败: %v", err) } var errors []string statusCodes = make([]int, 0, len(s.config.SearchEngines)) for _, engine := range s.config.SearchEngines { endpoint := fmt.Sprintf("%s/indexnow", engine) req, reqErr := http.NewRequest("POST", endpoint, bytes.NewBuffer(jsonData)) if reqErr != nil { errors = append(errors, fmt.Sprintf("创建请求失败: %v", reqErr)) continue } req.Header.Set("Content-Type", "application/json") resp, doErr := s.client.Do(req) if doErr != nil { errors = append(errors, fmt.Sprintf("请求 %s 失败: %v", engine, doErr)) continue } statusCodes = append(statusCodes, resp.StatusCode) log.Info("indexnow: api response", log.Any("engine", engine), log.Any("statusCode", resp.StatusCode), log.Any("urlCount", len(urls))) if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { errors = append(errors, fmt.Sprintf("搜索引擎 %s 返回错误状态码: %d", engine, resp.StatusCode)) } resp.Body.Close() } if len(errors) > 0 { return statusCodes, fmt.Errorf("%v", errors) } return statusCodes, nil } // splitIntoBatches 将URL列表分割成多个批次 func (s *BatchIndexNowSubmitter) splitIntoBatches(urls []string, batchSize int) [][]string { var batches [][]string for i := 0; i < len(urls); i += batchSize { end := i + batchSize if end > len(urls) { end = len(urls) } batches = append(batches, urls[i:end]) } return batches } // BatchResult 批量提交结果 type BatchResult struct { TotalURLs int SuccessCount int FailCount int Errors []string SuccessURLs []string StatusCodes []int // 各引擎 HTTP 状态码(按 SearchEngines 顺序,含重试后最后一次) } // String 结果字符串表示 func (r *BatchResult) String() string { return fmt.Sprintf("提交完成: 总数=%d, 成功=%d, 失败=%d, 错误数=%d, statusCodes=%v", r.TotalURLs, r.SuccessCount, r.FailCount, len(r.Errors), r.StatusCodes) }