247 lines
6.9 KiB
Go
247 lines
6.9 KiB
Go
// Package indexnow 实现 IndexNow 协议提交，推送到官方入口后由 Bing / Yandex 等共享数据。
|
|
// 调用方是 web 进程的 seopush，在文章发布 / 更新 / 删除时推送对应页面地址。
|
|
package indexnow
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
"videoweb/pub/common/log"
|
|
)
|
|
|
|
// DefaultKey IndexNow 默认密钥，与站点根目录下的 {key}.txt 验证文件对应
|
|
const DefaultKey = "6d41643810d540ce924edb982c3d3171"
|
|
|
|
// DefaultSearchEngine 官方入口，会自动分发给 Bing、Yandex 等搜索引擎
|
|
const DefaultSearchEngine = "https://api.indexnow.org"
|
|
|
|
// IndexNowRequest 提交请求结构体
|
|
type IndexNowRequest struct {
|
|
Host string `json:"host"`
|
|
Key string `json:"key"`
|
|
KeyLocation string `json:"keyLocation,omitempty"`
|
|
URLList []string `json:"urlList"`
|
|
}
|
|
|
|
// IndexNowConfig 配置结构体
|
|
type IndexNowConfig struct {
|
|
Key string
|
|
Host string
|
|
KeyLocation string
|
|
SearchEngines []string
|
|
RetryCount int
|
|
RetryDelay time.Duration
|
|
// BatchInterval 多批次之间的间隔，避免触发频率限制；单批提交不生效
|
|
BatchInterval time.Duration
|
|
}
|
|
|
|
// BuildConfig 依据站点地址推导 IndexNow 配置
|
|
func BuildConfig(webAddr string) (*IndexNowConfig, error) {
|
|
host, keyLocation, err := hostFromWebAddr(webAddr, DefaultKey)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &IndexNowConfig{
|
|
Key: DefaultKey,
|
|
Host: host,
|
|
KeyLocation: keyLocation,
|
|
SearchEngines: []string{DefaultSearchEngine},
|
|
RetryCount: 3,
|
|
RetryDelay: 2 * time.Second,
|
|
BatchInterval: 60 * time.Second,
|
|
}, nil
|
|
}
|
|
|
|
func hostFromWebAddr(webAddr, key string) (host, keyLocation string, err error) {
|
|
normalized := webAddr
|
|
if !strings.HasPrefix(normalized, "http://") && !strings.HasPrefix(normalized, "https://") {
|
|
normalized = "https://" + normalized
|
|
}
|
|
u, parseErr := url.Parse(normalized)
|
|
if parseErr != nil {
|
|
return "", "", parseErr
|
|
}
|
|
host = u.Hostname()
|
|
if host == "" {
|
|
return "", "", fmt.Errorf("invalid webAddr: %s", webAddr)
|
|
}
|
|
base := strings.TrimSuffix(normalized, "/")
|
|
keyLocation = fmt.Sprintf("%s/%s.txt", base, key)
|
|
return host, keyLocation, nil
|
|
}
|
|
|
|
// BatchIndexNowSubmitter 批量提交器
|
|
type BatchIndexNowSubmitter struct {
|
|
config *IndexNowConfig
|
|
client *http.Client
|
|
}
|
|
|
|
// NewBatchIndexNowSubmitter 创建新的批量提交器
|
|
func NewBatchIndexNowSubmitter(config *IndexNowConfig) *BatchIndexNowSubmitter {
|
|
if config.RetryCount == 0 {
|
|
config.RetryCount = 3
|
|
}
|
|
if config.RetryDelay == 0 {
|
|
config.RetryDelay = 1 * time.Second
|
|
}
|
|
if config.BatchInterval == 0 {
|
|
config.BatchInterval = 60 * time.Second
|
|
}
|
|
|
|
return &BatchIndexNowSubmitter{
|
|
config: config,
|
|
client: &http.Client{
|
|
Timeout: 30 * time.Second,
|
|
},
|
|
}
|
|
}
|
|
|
|
// SubmitBatch 批量提交URL，返回成功提交的 URL 列表
|
|
func (s *BatchIndexNowSubmitter) SubmitBatch(urls []string) *BatchResult {
|
|
result := &BatchResult{
|
|
TotalURLs: len(urls),
|
|
SuccessCount: 0,
|
|
FailCount: 0,
|
|
Errors: make([]string, 0),
|
|
SuccessURLs: make([]string, 0),
|
|
StatusCodes: make([]int, 0),
|
|
}
|
|
|
|
if len(urls) == 0 {
|
|
return result
|
|
}
|
|
|
|
// 分批处理，每批最多100个URL（避免单次请求过大）
|
|
const batchSize = 100
|
|
batches := s.splitIntoBatches(urls, batchSize)
|
|
for batchIndex, batchURLs := range batches {
|
|
codes, err := s.submitWithRetry(batchURLs)
|
|
result.StatusCodes = append(result.StatusCodes, codes...)
|
|
if err != nil {
|
|
result.FailCount += len(batchURLs)
|
|
result.Errors = append(result.Errors,
|
|
fmt.Sprintf("批次 %d 提交失败: %v", batchIndex+1, err))
|
|
} else {
|
|
result.SuccessCount += len(batchURLs)
|
|
result.SuccessURLs = append(result.SuccessURLs, batchURLs...)
|
|
}
|
|
if batchIndex < len(batches)-1 {
|
|
time.Sleep(s.config.BatchInterval)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// submitWithRetry 带重试的提交，返回各引擎 HTTP 状态码
|
|
func (s *BatchIndexNowSubmitter) submitWithRetry(urls []string) (statusCodes []int, err error) {
|
|
var lastError error
|
|
var lastCodes []int
|
|
|
|
for attempt := 1; attempt <= s.config.RetryCount; attempt++ {
|
|
codes, submitErr := s.submitToAllEngines(urls)
|
|
lastCodes = codes
|
|
if submitErr != nil {
|
|
lastError = submitErr
|
|
if attempt < s.config.RetryCount {
|
|
log.Warn("indexnow: submit retry",
|
|
log.Any("attempt", attempt),
|
|
log.Any("delay", s.config.RetryDelay),
|
|
log.Any("statusCodes", codes),
|
|
log.E(submitErr))
|
|
time.Sleep(s.config.RetryDelay)
|
|
continue
|
|
}
|
|
} else {
|
|
return codes, nil
|
|
}
|
|
}
|
|
|
|
return lastCodes, fmt.Errorf("经过 %d 次重试后仍然失败: %v", s.config.RetryCount, lastError)
|
|
}
|
|
|
|
// submitToAllEngines 向所有搜索引擎提交，返回各引擎 HTTP 状态码
|
|
func (s *BatchIndexNowSubmitter) submitToAllEngines(urls []string) (statusCodes []int, err error) {
|
|
requestBody := IndexNowRequest{
|
|
Host: s.config.Host,
|
|
Key: s.config.Key,
|
|
KeyLocation: s.config.KeyLocation,
|
|
URLList: urls,
|
|
}
|
|
|
|
jsonData, err := json.Marshal(requestBody)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("JSON编码失败: %v", err)
|
|
}
|
|
|
|
var errors []string
|
|
statusCodes = make([]int, 0, len(s.config.SearchEngines))
|
|
for _, engine := range s.config.SearchEngines {
|
|
endpoint := fmt.Sprintf("%s/indexnow", engine)
|
|
|
|
req, reqErr := http.NewRequest("POST", endpoint, bytes.NewBuffer(jsonData))
|
|
if reqErr != nil {
|
|
errors = append(errors, fmt.Sprintf("创建请求失败: %v", reqErr))
|
|
continue
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, doErr := s.client.Do(req)
|
|
if doErr != nil {
|
|
errors = append(errors, fmt.Sprintf("请求 %s 失败: %v", engine, doErr))
|
|
continue
|
|
}
|
|
|
|
statusCodes = append(statusCodes, resp.StatusCode)
|
|
log.Info("indexnow: api response",
|
|
log.Any("engine", engine),
|
|
log.Any("statusCode", resp.StatusCode),
|
|
log.Any("urlCount", len(urls)))
|
|
|
|
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
|
|
errors = append(errors, fmt.Sprintf("搜索引擎 %s 返回错误状态码: %d", engine, resp.StatusCode))
|
|
}
|
|
resp.Body.Close()
|
|
}
|
|
|
|
if len(errors) > 0 {
|
|
return statusCodes, fmt.Errorf("%v", errors)
|
|
}
|
|
|
|
return statusCodes, nil
|
|
}
|
|
|
|
// splitIntoBatches 将URL列表分割成多个批次
|
|
func (s *BatchIndexNowSubmitter) splitIntoBatches(urls []string, batchSize int) [][]string {
|
|
var batches [][]string
|
|
|
|
for i := 0; i < len(urls); i += batchSize {
|
|
end := i + batchSize
|
|
if end > len(urls) {
|
|
end = len(urls)
|
|
}
|
|
batches = append(batches, urls[i:end])
|
|
}
|
|
|
|
return batches
|
|
}
|
|
|
|
// BatchResult 批量提交结果
|
|
type BatchResult struct {
|
|
TotalURLs int
|
|
SuccessCount int
|
|
FailCount int
|
|
Errors []string
|
|
SuccessURLs []string
|
|
StatusCodes []int // 各引擎 HTTP 状态码（按 SearchEngines 顺序，含重试后最后一次）
|
|
}
|
|
|
|
// String 结果字符串表示
|
|
func (r *BatchResult) String() string {
|
|
return fmt.Sprintf("提交完成: 总数=%d, 成功=%d, 失败=%d, 错误数=%d, statusCodes=%v",
|
|
r.TotalURLs, r.SuccessCount, r.FailCount, len(r.Errors), r.StatusCodes)
|
|
}
|