web_huakui_server/pub/seo/indexnow/submitter.go

// Package indexnow 实现 IndexNow 协议提交，推送到官方入口后由 Bing / Yandex 等共享数据。
// 调用方是 web 进程的 seopush，在文章发布 / 更新 / 删除时推送对应页面地址。
package indexnow

import (
	"bytes"
	"encoding/json"
	"fmt"
	"hkdj-web-server/pub/common/log"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// DefaultSearchEngine 官方入口，会自动分发给 Bing、Yandex 等搜索引擎
const DefaultSearchEngine = "https://api.indexnow.org"

// IndexNowRequest 提交请求结构体
type IndexNowRequest struct {
	Host        string   `json:"host"`
	Key         string   `json:"key"`
	KeyLocation string   `json:"keyLocation,omitempty"`
	URLList     []string `json:"urlList"`
}

// IndexNowConfig 配置结构体
type IndexNowConfig struct {
	Key           string
	Host          string
	KeyLocation   string
	SearchEngines []string
	RetryCount    int
	RetryDelay    time.Duration
	// BatchInterval 多批次之间的间隔，避免触发频率限制；单批提交不生效
	BatchInterval time.Duration
}

// BuildConfig 依据站点地址和调用方提供的验证 key 推导 IndexNow 配置。
func BuildConfig(webAddr, key string) (*IndexNowConfig, error) {
	key = strings.TrimSpace(key)
	if key == "" {
		return nil, fmt.Errorf("IndexNow key is not configured")
	}
	host, keyLocation, err := hostFromWebAddr(webAddr, key)
	if err != nil {
		return nil, err
	}
	return &IndexNowConfig{
		Key:           key,
		Host:          host,
		KeyLocation:   keyLocation,
		SearchEngines: []string{DefaultSearchEngine},
		RetryCount:    3,
		RetryDelay:    2 * time.Second,
		BatchInterval: 60 * time.Second,
	}, nil
}

func hostFromWebAddr(webAddr, key string) (host, keyLocation string, err error) {
	normalized := webAddr
	if !strings.HasPrefix(normalized, "http://") && !strings.HasPrefix(normalized, "https://") {
		normalized = "https://" + normalized
	}
	u, parseErr := url.Parse(normalized)
	if parseErr != nil {
		return "", "", parseErr
	}
	host = u.Hostname()
	if host == "" {
		return "", "", fmt.Errorf("invalid webAddr: %s", webAddr)
	}
	base := strings.TrimSuffix(normalized, "/")
	keyLocation = fmt.Sprintf("%s/%s.txt", base, key)
	return host, keyLocation, nil
}

// BatchIndexNowSubmitter 批量提交器
type BatchIndexNowSubmitter struct {
	config *IndexNowConfig
	client *http.Client
}

// NewBatchIndexNowSubmitter 创建新的批量提交器
func NewBatchIndexNowSubmitter(config *IndexNowConfig) *BatchIndexNowSubmitter {
	if config.RetryCount == 0 {
		config.RetryCount = 3
	}
	if config.RetryDelay == 0 {
		config.RetryDelay = 1 * time.Second
	}
	if config.BatchInterval == 0 {
		config.BatchInterval = 60 * time.Second
	}

	return &BatchIndexNowSubmitter{
		config: config,
		client: &http.Client{
			Timeout: 30 * time.Second,
		},
	}
}

// SubmitBatch 批量提交URL，返回成功提交的 URL 列表
func (s *BatchIndexNowSubmitter) SubmitBatch(urls []string) *BatchResult {
	result := &BatchResult{
		TotalURLs:    len(urls),
		SuccessCount: 0,
		FailCount:    0,
		Errors:       make([]string, 0),
		SuccessURLs:  make([]string, 0),
		StatusCodes:  make([]int, 0),
	}

	if len(urls) == 0 {
		return result
	}

	// 分批处理，每批最多100个URL（避免单次请求过大）
	const batchSize = 100
	batches := s.splitIntoBatches(urls, batchSize)
	for batchIndex, batchURLs := range batches {
		codes, err := s.submitWithRetry(batchURLs)
		result.StatusCodes = append(result.StatusCodes, codes...)
		if err != nil {
			result.FailCount += len(batchURLs)
			result.Errors = append(result.Errors,
				fmt.Sprintf("批次 %d 提交失败: %v", batchIndex+1, err))
		} else {
			result.SuccessCount += len(batchURLs)
			result.SuccessURLs = append(result.SuccessURLs, batchURLs...)
		}
		if batchIndex < len(batches)-1 {
			time.Sleep(s.config.BatchInterval)
		}
	}
	return result
}

// submitWithRetry 带重试的提交，返回各引擎 HTTP 状态码
func (s *BatchIndexNowSubmitter) submitWithRetry(urls []string) (statusCodes []int, err error) {
	var lastError error
	var lastCodes []int

	for attempt := 1; attempt <= s.config.RetryCount; attempt++ {
		codes, submitErr := s.submitToAllEngines(urls)
		lastCodes = codes
		if submitErr != nil {
			lastError = submitErr
			if attempt < s.config.RetryCount {
				log.Warn("indexnow: submit retry",
					log.Any("attempt", attempt),
					log.Any("delay", s.config.RetryDelay),
					log.Any("statusCodes", codes),
					log.E(submitErr))
				time.Sleep(s.config.RetryDelay)
				continue
			}
		} else {
			return codes, nil
		}
	}

	return lastCodes, fmt.Errorf("经过 %d 次重试后仍然失败: %v", s.config.RetryCount, lastError)
}

// submitToAllEngines 向所有搜索引擎提交，返回各引擎 HTTP 状态码
func (s *BatchIndexNowSubmitter) submitToAllEngines(urls []string) (statusCodes []int, err error) {
	requestBody := IndexNowRequest{
		Host:        s.config.Host,
		Key:         s.config.Key,
		KeyLocation: s.config.KeyLocation,
		URLList:     urls,
	}

	jsonData, err := json.Marshal(requestBody)
	if err != nil {
		return nil, fmt.Errorf("JSON编码失败: %v", err)
	}

	var errors []string
	statusCodes = make([]int, 0, len(s.config.SearchEngines))
	for _, engine := range s.config.SearchEngines {
		endpoint := fmt.Sprintf("%s/indexnow", engine)

		req, reqErr := http.NewRequest("POST", endpoint, bytes.NewBuffer(jsonData))
		if reqErr != nil {
			errors = append(errors, fmt.Sprintf("创建请求失败: %v", reqErr))
			continue
		}
		req.Header.Set("Content-Type", "application/json")

		resp, doErr := s.client.Do(req)
		if doErr != nil {
			errors = append(errors, fmt.Sprintf("请求 %s 失败: %v", engine, doErr))
			continue
		}

		statusCodes = append(statusCodes, resp.StatusCode)
		log.Info("indexnow: api response",
			log.Any("engine", engine),
			log.Any("statusCode", resp.StatusCode),
			log.Any("urlCount", len(urls)))

		if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
			errors = append(errors, fmt.Sprintf("搜索引擎 %s 返回错误状态码: %d", engine, resp.StatusCode))
		}
		resp.Body.Close()
	}

	if len(errors) > 0 {
		return statusCodes, fmt.Errorf("%v", errors)
	}

	return statusCodes, nil
}

// splitIntoBatches 将URL列表分割成多个批次
func (s *BatchIndexNowSubmitter) splitIntoBatches(urls []string, batchSize int) [][]string {
	var batches [][]string

	for i := 0; i < len(urls); i += batchSize {
		end := i + batchSize
		if end > len(urls) {
			end = len(urls)
		}
		batches = append(batches, urls[i:end])
	}

	return batches
}

// BatchResult 批量提交结果
type BatchResult struct {
	TotalURLs    int
	SuccessCount int
	FailCount    int
	Errors       []string
	SuccessURLs  []string
	StatusCodes  []int // 各引擎 HTTP 状态码（按 SearchEngines 顺序，含重试后最后一次）
}

// String 结果字符串表示
func (r *BatchResult) String() string {
	return fmt.Sprintf("提交完成: 总数=%d, 成功=%d, 失败=%d, 错误数=%d, statusCodes=%v",
		r.TotalURLs, r.SuccessCount, r.FailCount, len(r.Errors), r.StatusCodes)
}