Initial commit: web_hghl_server
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
@@ -0,0 +1,246 @@
|
||||
// Package indexnow 实现 IndexNow 协议提交，推送到官方入口后由 Bing / Yandex 等共享数据。
|
||||
// 调用方是 web 进程的 seopush，在文章发布 / 更新 / 删除时推送对应页面地址。
|
||||
package indexnow
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
)
|
||||
|
||||
// DefaultKey IndexNow 默认密钥，与站点根目录下的 {key}.txt 验证文件对应
|
||||
const DefaultKey = "6d41643810d540ce924edb982c3d3171"
|
||||
|
||||
// DefaultSearchEngine 官方入口，会自动分发给 Bing、Yandex 等搜索引擎
|
||||
const DefaultSearchEngine = "https://api.indexnow.org"
|
||||
|
||||
// IndexNowRequest 提交请求结构体
|
||||
type IndexNowRequest struct {
|
||||
Host string `json:"host"`
|
||||
Key string `json:"key"`
|
||||
KeyLocation string `json:"keyLocation,omitempty"`
|
||||
URLList []string `json:"urlList"`
|
||||
}
|
||||
|
||||
// IndexNowConfig 配置结构体
|
||||
type IndexNowConfig struct {
|
||||
Key string
|
||||
Host string
|
||||
KeyLocation string
|
||||
SearchEngines []string
|
||||
RetryCount int
|
||||
RetryDelay time.Duration
|
||||
// BatchInterval 多批次之间的间隔，避免触发频率限制；单批提交不生效
|
||||
BatchInterval time.Duration
|
||||
}
|
||||
|
||||
// BuildConfig 依据站点地址推导 IndexNow 配置
|
||||
func BuildConfig(webAddr string) (*IndexNowConfig, error) {
|
||||
host, keyLocation, err := hostFromWebAddr(webAddr, DefaultKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &IndexNowConfig{
|
||||
Key: DefaultKey,
|
||||
Host: host,
|
||||
KeyLocation: keyLocation,
|
||||
SearchEngines: []string{DefaultSearchEngine},
|
||||
RetryCount: 3,
|
||||
RetryDelay: 2 * time.Second,
|
||||
BatchInterval: 60 * time.Second,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func hostFromWebAddr(webAddr, key string) (host, keyLocation string, err error) {
|
||||
normalized := webAddr
|
||||
if !strings.HasPrefix(normalized, "http://") && !strings.HasPrefix(normalized, "https://") {
|
||||
normalized = "https://" + normalized
|
||||
}
|
||||
u, parseErr := url.Parse(normalized)
|
||||
if parseErr != nil {
|
||||
return "", "", parseErr
|
||||
}
|
||||
host = u.Hostname()
|
||||
if host == "" {
|
||||
return "", "", fmt.Errorf("invalid webAddr: %s", webAddr)
|
||||
}
|
||||
base := strings.TrimSuffix(normalized, "/")
|
||||
keyLocation = fmt.Sprintf("%s/%s.txt", base, key)
|
||||
return host, keyLocation, nil
|
||||
}
|
||||
|
||||
// BatchIndexNowSubmitter 批量提交器
|
||||
type BatchIndexNowSubmitter struct {
|
||||
config *IndexNowConfig
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewBatchIndexNowSubmitter 创建新的批量提交器
|
||||
func NewBatchIndexNowSubmitter(config *IndexNowConfig) *BatchIndexNowSubmitter {
|
||||
if config.RetryCount == 0 {
|
||||
config.RetryCount = 3
|
||||
}
|
||||
if config.RetryDelay == 0 {
|
||||
config.RetryDelay = 1 * time.Second
|
||||
}
|
||||
if config.BatchInterval == 0 {
|
||||
config.BatchInterval = 60 * time.Second
|
||||
}
|
||||
|
||||
return &BatchIndexNowSubmitter{
|
||||
config: config,
|
||||
client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// SubmitBatch 批量提交URL，返回成功提交的 URL 列表
|
||||
func (s *BatchIndexNowSubmitter) SubmitBatch(urls []string) *BatchResult {
|
||||
result := &BatchResult{
|
||||
TotalURLs: len(urls),
|
||||
SuccessCount: 0,
|
||||
FailCount: 0,
|
||||
Errors: make([]string, 0),
|
||||
SuccessURLs: make([]string, 0),
|
||||
StatusCodes: make([]int, 0),
|
||||
}
|
||||
|
||||
if len(urls) == 0 {
|
||||
return result
|
||||
}
|
||||
|
||||
// 分批处理，每批最多100个URL（避免单次请求过大）
|
||||
const batchSize = 100
|
||||
batches := s.splitIntoBatches(urls, batchSize)
|
||||
for batchIndex, batchURLs := range batches {
|
||||
codes, err := s.submitWithRetry(batchURLs)
|
||||
result.StatusCodes = append(result.StatusCodes, codes...)
|
||||
if err != nil {
|
||||
result.FailCount += len(batchURLs)
|
||||
result.Errors = append(result.Errors,
|
||||
fmt.Sprintf("批次 %d 提交失败: %v", batchIndex+1, err))
|
||||
} else {
|
||||
result.SuccessCount += len(batchURLs)
|
||||
result.SuccessURLs = append(result.SuccessURLs, batchURLs...)
|
||||
}
|
||||
if batchIndex < len(batches)-1 {
|
||||
time.Sleep(s.config.BatchInterval)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// submitWithRetry 带重试的提交，返回各引擎 HTTP 状态码
|
||||
func (s *BatchIndexNowSubmitter) submitWithRetry(urls []string) (statusCodes []int, err error) {
|
||||
var lastError error
|
||||
var lastCodes []int
|
||||
|
||||
for attempt := 1; attempt <= s.config.RetryCount; attempt++ {
|
||||
codes, submitErr := s.submitToAllEngines(urls)
|
||||
lastCodes = codes
|
||||
if submitErr != nil {
|
||||
lastError = submitErr
|
||||
if attempt < s.config.RetryCount {
|
||||
log.Warn("indexnow: submit retry",
|
||||
log.Any("attempt", attempt),
|
||||
log.Any("delay", s.config.RetryDelay),
|
||||
log.Any("statusCodes", codes),
|
||||
log.E(submitErr))
|
||||
time.Sleep(s.config.RetryDelay)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
return codes, nil
|
||||
}
|
||||
}
|
||||
|
||||
return lastCodes, fmt.Errorf("经过 %d 次重试后仍然失败: %v", s.config.RetryCount, lastError)
|
||||
}
|
||||
|
||||
// submitToAllEngines 向所有搜索引擎提交，返回各引擎 HTTP 状态码
|
||||
func (s *BatchIndexNowSubmitter) submitToAllEngines(urls []string) (statusCodes []int, err error) {
|
||||
requestBody := IndexNowRequest{
|
||||
Host: s.config.Host,
|
||||
Key: s.config.Key,
|
||||
KeyLocation: s.config.KeyLocation,
|
||||
URLList: urls,
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("JSON编码失败: %v", err)
|
||||
}
|
||||
|
||||
var errors []string
|
||||
statusCodes = make([]int, 0, len(s.config.SearchEngines))
|
||||
for _, engine := range s.config.SearchEngines {
|
||||
endpoint := fmt.Sprintf("%s/indexnow", engine)
|
||||
|
||||
req, reqErr := http.NewRequest("POST", endpoint, bytes.NewBuffer(jsonData))
|
||||
if reqErr != nil {
|
||||
errors = append(errors, fmt.Sprintf("创建请求失败: %v", reqErr))
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, doErr := s.client.Do(req)
|
||||
if doErr != nil {
|
||||
errors = append(errors, fmt.Sprintf("请求 %s 失败: %v", engine, doErr))
|
||||
continue
|
||||
}
|
||||
|
||||
statusCodes = append(statusCodes, resp.StatusCode)
|
||||
log.Info("indexnow: api response",
|
||||
log.Any("engine", engine),
|
||||
log.Any("statusCode", resp.StatusCode),
|
||||
log.Any("urlCount", len(urls)))
|
||||
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
|
||||
errors = append(errors, fmt.Sprintf("搜索引擎 %s 返回错误状态码: %d", engine, resp.StatusCode))
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
return statusCodes, fmt.Errorf("%v", errors)
|
||||
}
|
||||
|
||||
return statusCodes, nil
|
||||
}
|
||||
|
||||
// splitIntoBatches 将URL列表分割成多个批次
|
||||
func (s *BatchIndexNowSubmitter) splitIntoBatches(urls []string, batchSize int) [][]string {
|
||||
var batches [][]string
|
||||
|
||||
for i := 0; i < len(urls); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(urls) {
|
||||
end = len(urls)
|
||||
}
|
||||
batches = append(batches, urls[i:end])
|
||||
}
|
||||
|
||||
return batches
|
||||
}
|
||||
|
||||
// BatchResult 批量提交结果
|
||||
type BatchResult struct {
|
||||
TotalURLs int
|
||||
SuccessCount int
|
||||
FailCount int
|
||||
Errors []string
|
||||
SuccessURLs []string
|
||||
StatusCodes []int // 各引擎 HTTP 状态码（按 SearchEngines 顺序，含重试后最后一次）
|
||||
}
|
||||
|
||||
// String 结果字符串表示
|
||||
func (r *BatchResult) String() string {
|
||||
return fmt.Sprintf("提交完成: 总数=%d, 成功=%d, 失败=%d, 错误数=%d, statusCodes=%v",
|
||||
r.TotalURLs, r.SuccessCount, r.FailCount, len(r.Errors), r.StatusCodes)
|
||||
}
|
||||
Reference in New Issue
Block a user