Initial commit: web_hghl_server

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-10-26 16:28:14 +08:00
commit 65fca60622
653 changed files with 135901 additions and 0 deletions
@@ -0,0 +1,187 @@
+// Package googleindex 对接 Google Indexing API，在文章发布/更新/删除时推送具体页面 URL。
+// 需要在 Google Cloud Console 创建服务账号并开启 Indexing API，
+// 同时在 Search Console 中把该服务账号邮箱添加为站点「所有者」。
+package googleindex
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+	"videoweb/pub/common/log"
+
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/indexing/v3"
+	"google.golang.org/api/option"
+)
+
+// 推送动作类型
+const (
+	TypeUpdated = "URL_UPDATED" // 新增或更新
+	TypeDeleted = "URL_DELETED" // 已删除
+)
+
+// defaultDailyQuota Google Indexing API 默认每天 200 个 URL 请求
+const defaultDailyQuota = int64(200)
+
+// ErrQuotaExceeded 配额已用尽，可能来自本地计数拦截，也可能是 Google 返回的 429。
+// 两者都只代表「现在不能推」，调用方应保留任务等下次，而不是计为失败重试。
+var ErrQuotaExceeded = errors.New("googleindex: daily quota exceeded")
+
+// ErrNotEnabled 未启用或未初始化
+var ErrNotEnabled = errors.New("googleindex: not enabled")
+
+// Client Indexing API 客户端，内置每日配额保护
+type Client struct {
+	svc   *indexing.Service
+	quota *quotaGuard
+}
+
+var (
+	defaultClient *Client
+	defaultMu     sync.RWMutex
+)
+
+// Init 初始化全局客户端，未启用时不报错，仅记录日志
+func Init(ctx context.Context) {
+	client, err := NewClient(ctx)
+	if err != nil {
+		log.Error("googleindex: init failed", log.E(err))
+		return
+	}
+	defaultMu.Lock()
+	defaultClient = client
+	defaultMu.Unlock()
+	log.Info("googleindex: ready", log.Any("dailyQuota", client.quota.limit))
+}
+
+// NewClient 依据配置创建客户端
+func NewClient(ctx context.Context) (*Client, error) {
+	credentials, err := loadCredentials()
+	if err != nil {
+		return nil, err
+	}
+	svc, err := indexing.NewService(ctx, option.WithCredentialsJSON(credentials))
+	if err != nil {
+		return nil, fmt.Errorf("create indexing service failed: %w", err)
+	}
+	limit := defaultDailyQuota
+	return &Client{svc: svc, quota: &quotaGuard{limit: limit}}, nil
+}
+
+func loadCredentials() ([]byte, error) {
+	credentialsFile := "composed-anvil-470512-r8-fd674ac0d7dc.json"
+	b, err := os.ReadFile(credentialsFile)
+	if err != nil {
+		return nil, fmt.Errorf("read credentials file %s failed: %w", credentialsFile, err)
+	}
+	return b, nil
+}
+
+// Publish 使用全局客户端推送单个 URL
+func Publish(ctx context.Context, pageURL, notifyType string) (*indexing.PublishUrlNotificationResponse, error) {
+	defaultMu.RLock()
+	client := defaultClient
+	defaultMu.RUnlock()
+	if client == nil {
+		return nil, ErrNotEnabled
+	}
+	return client.Publish(ctx, pageURL, notifyType)
+}
+
+// Publish 推送单个 URL，notifyType 取 TypeUpdated / TypeDeleted
+func (c *Client) Publish(ctx context.Context, pageURL, notifyType string) (*indexing.PublishUrlNotificationResponse, error) {
+	if strings.TrimSpace(pageURL) == "" {
+		return nil, errors.New("googleindex: empty url")
+	}
+	if notifyType != TypeUpdated && notifyType != TypeDeleted {
+		return nil, fmt.Errorf("googleindex: invalid notify type %s", notifyType)
+	}
+	if !c.quota.acquire() {
+		return nil, ErrQuotaExceeded
+	}
+
+	resp, err := c.svc.UrlNotifications.Publish(&indexing.UrlNotification{
+		Url:  pageURL,
+		Type: notifyType,
+	}).Context(ctx).Do()
+	if err != nil {
+		// Google 已明确告知配额用尽，说明本地计数比实际用量宽松：
+		// 本地日界按服务器时区计算、且计数在进程重启后归零，都可能早于 Google 放行。
+		// 此时把本地计数拉满，避免后续请求继续无效地打过去。
+		if isQuotaError(err) {
+			c.quota.exhaust()
+			return resp, fmt.Errorf("%w: %v", ErrQuotaExceeded, err)
+		}
+		// 请求已消耗配额但失败，退回本地计数，避免本地计数与实际用量偏差过大
+		c.quota.release()
+		return resp, err
+	}
+	return resp, nil
+}
+
+// isQuotaError 判断错误是否为配额或频率限制。
+// 这类错误只代表「现在不能推」，调用方应保留任务等下次，而不是计为失败重试。
+// 注意 403 还用于站点所有权不足等需要人工介入的场景，因此只在 reason 明确是配额时才算。
+func isQuotaError(err error) bool {
+	var gerr *googleapi.Error
+	if !errors.As(err, &gerr) {
+		return false
+	}
+	if gerr.Code == http.StatusTooManyRequests {
+		return true
+	}
+	for _, item := range gerr.Errors {
+		switch item.Reason {
+		case "rateLimitExceeded", "quotaExceeded", "userRateLimitExceeded":
+			return true
+		}
+	}
+	return false
+}
+
+// quotaGuard 按自然日计数的配额保护。
+// 计数仅存于内存，进程重启会归零；真正超配额时 Google 会返回 429，由调用方按失败重试处理。
+type quotaGuard struct {
+	mu    sync.Mutex
+	limit int64
+	day   string
+	used  int64
+}
+
+func (q *quotaGuard) acquire() bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	today := time.Now().Format("2006-01-02")
+	if q.day != today {
+		q.day = today
+		q.used = 0
+	}
+	if q.used >= q.limit {
+		return false
+	}
+	q.used++
+	return true
+}
+
+func (q *quotaGuard) release() {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	if q.used > 0 {
+		q.used--
+	}
+}
+
+// exhaust 把当天计数置满，用于 Google 已返回配额超限时同步本地状态。
+// 代价是本地要等到下一个自然日才放行，可能晚于 Google 实际的配额重置时刻；
+// 换来的是不再持续发送必定失败的请求。
+func (q *quotaGuard) exhaust() {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	q.day = time.Now().Format("2006-01-02")
+	q.used = q.limit
+}
@@ -0,0 +1,246 @@
+// Package indexnow 实现 IndexNow 协议提交，推送到官方入口后由 Bing / Yandex 等共享数据。
+// 调用方是 web 进程的 seopush，在文章发布 / 更新 / 删除时推送对应页面地址。
+package indexnow
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+	"videoweb/pub/common/log"
+)
+
+// DefaultKey IndexNow 默认密钥，与站点根目录下的 {key}.txt 验证文件对应
+const DefaultKey = "6d41643810d540ce924edb982c3d3171"
+
+// DefaultSearchEngine 官方入口，会自动分发给 Bing、Yandex 等搜索引擎
+const DefaultSearchEngine = "https://api.indexnow.org"
+
+// IndexNowRequest 提交请求结构体
+type IndexNowRequest struct {
+	Host        string   `json:"host"`
+	Key         string   `json:"key"`
+	KeyLocation string   `json:"keyLocation,omitempty"`
+	URLList     []string `json:"urlList"`
+}
+
+// IndexNowConfig 配置结构体
+type IndexNowConfig struct {
+	Key           string
+	Host          string
+	KeyLocation   string
+	SearchEngines []string
+	RetryCount    int
+	RetryDelay    time.Duration
+	// BatchInterval 多批次之间的间隔，避免触发频率限制；单批提交不生效
+	BatchInterval time.Duration
+}
+
+// BuildConfig 依据站点地址推导 IndexNow 配置
+func BuildConfig(webAddr string) (*IndexNowConfig, error) {
+	host, keyLocation, err := hostFromWebAddr(webAddr, DefaultKey)
+	if err != nil {
+		return nil, err
+	}
+	return &IndexNowConfig{
+		Key:           DefaultKey,
+		Host:          host,
+		KeyLocation:   keyLocation,
+		SearchEngines: []string{DefaultSearchEngine},
+		RetryCount:    3,
+		RetryDelay:    2 * time.Second,
+		BatchInterval: 60 * time.Second,
+	}, nil
+}
+
+func hostFromWebAddr(webAddr, key string) (host, keyLocation string, err error) {
+	normalized := webAddr
+	if !strings.HasPrefix(normalized, "http://") && !strings.HasPrefix(normalized, "https://") {
+		normalized = "https://" + normalized
+	}
+	u, parseErr := url.Parse(normalized)
+	if parseErr != nil {
+		return "", "", parseErr
+	}
+	host = u.Hostname()
+	if host == "" {
+		return "", "", fmt.Errorf("invalid webAddr: %s", webAddr)
+	}
+	base := strings.TrimSuffix(normalized, "/")
+	keyLocation = fmt.Sprintf("%s/%s.txt", base, key)
+	return host, keyLocation, nil
+}
+
+// BatchIndexNowSubmitter 批量提交器
+type BatchIndexNowSubmitter struct {
+	config *IndexNowConfig
+	client *http.Client
+}
+
+// NewBatchIndexNowSubmitter 创建新的批量提交器
+func NewBatchIndexNowSubmitter(config *IndexNowConfig) *BatchIndexNowSubmitter {
+	if config.RetryCount == 0 {
+		config.RetryCount = 3
+	}
+	if config.RetryDelay == 0 {
+		config.RetryDelay = 1 * time.Second
+	}
+	if config.BatchInterval == 0 {
+		config.BatchInterval = 60 * time.Second
+	}
+
+	return &BatchIndexNowSubmitter{
+		config: config,
+		client: &http.Client{
+			Timeout: 30 * time.Second,
+		},
+	}
+}
+
+// SubmitBatch 批量提交URL，返回成功提交的 URL 列表
+func (s *BatchIndexNowSubmitter) SubmitBatch(urls []string) *BatchResult {
+	result := &BatchResult{
+		TotalURLs:    len(urls),
+		SuccessCount: 0,
+		FailCount:    0,
+		Errors:       make([]string, 0),
+		SuccessURLs:  make([]string, 0),
+		StatusCodes:  make([]int, 0),
+	}
+
+	if len(urls) == 0 {
+		return result
+	}
+
+	// 分批处理，每批最多100个URL（避免单次请求过大）
+	const batchSize = 100
+	batches := s.splitIntoBatches(urls, batchSize)
+	for batchIndex, batchURLs := range batches {
+		codes, err := s.submitWithRetry(batchURLs)
+		result.StatusCodes = append(result.StatusCodes, codes...)
+		if err != nil {
+			result.FailCount += len(batchURLs)
+			result.Errors = append(result.Errors,
+				fmt.Sprintf("批次 %d 提交失败: %v", batchIndex+1, err))
+		} else {
+			result.SuccessCount += len(batchURLs)
+			result.SuccessURLs = append(result.SuccessURLs, batchURLs...)
+		}
+		if batchIndex < len(batches)-1 {
+			time.Sleep(s.config.BatchInterval)
+		}
+	}
+	return result
+}
+
+// submitWithRetry 带重试的提交，返回各引擎 HTTP 状态码
+func (s *BatchIndexNowSubmitter) submitWithRetry(urls []string) (statusCodes []int, err error) {
+	var lastError error
+	var lastCodes []int
+
+	for attempt := 1; attempt <= s.config.RetryCount; attempt++ {
+		codes, submitErr := s.submitToAllEngines(urls)
+		lastCodes = codes
+		if submitErr != nil {
+			lastError = submitErr
+			if attempt < s.config.RetryCount {
+				log.Warn("indexnow: submit retry",
+					log.Any("attempt", attempt),
+					log.Any("delay", s.config.RetryDelay),
+					log.Any("statusCodes", codes),
+					log.E(submitErr))
+				time.Sleep(s.config.RetryDelay)
+				continue
+			}
+		} else {
+			return codes, nil
+		}
+	}
+
+	return lastCodes, fmt.Errorf("经过 %d 次重试后仍然失败: %v", s.config.RetryCount, lastError)
+}
+
+// submitToAllEngines 向所有搜索引擎提交，返回各引擎 HTTP 状态码
+func (s *BatchIndexNowSubmitter) submitToAllEngines(urls []string) (statusCodes []int, err error) {
+	requestBody := IndexNowRequest{
+		Host:        s.config.Host,
+		Key:         s.config.Key,
+		KeyLocation: s.config.KeyLocation,
+		URLList:     urls,
+	}
+
+	jsonData, err := json.Marshal(requestBody)
+	if err != nil {
+		return nil, fmt.Errorf("JSON编码失败: %v", err)
+	}
+
+	var errors []string
+	statusCodes = make([]int, 0, len(s.config.SearchEngines))
+	for _, engine := range s.config.SearchEngines {
+		endpoint := fmt.Sprintf("%s/indexnow", engine)
+
+		req, reqErr := http.NewRequest("POST", endpoint, bytes.NewBuffer(jsonData))
+		if reqErr != nil {
+			errors = append(errors, fmt.Sprintf("创建请求失败: %v", reqErr))
+			continue
+		}
+		req.Header.Set("Content-Type", "application/json")
+
+		resp, doErr := s.client.Do(req)
+		if doErr != nil {
+			errors = append(errors, fmt.Sprintf("请求 %s 失败: %v", engine, doErr))
+			continue
+		}
+
+		statusCodes = append(statusCodes, resp.StatusCode)
+		log.Info("indexnow: api response",
+			log.Any("engine", engine),
+			log.Any("statusCode", resp.StatusCode),
+			log.Any("urlCount", len(urls)))
+
+		if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
+			errors = append(errors, fmt.Sprintf("搜索引擎 %s 返回错误状态码: %d", engine, resp.StatusCode))
+		}
+		resp.Body.Close()
+	}
+
+	if len(errors) > 0 {
+		return statusCodes, fmt.Errorf("%v", errors)
+	}
+
+	return statusCodes, nil
+}
+
+// splitIntoBatches 将URL列表分割成多个批次
+func (s *BatchIndexNowSubmitter) splitIntoBatches(urls []string, batchSize int) [][]string {
+	var batches [][]string
+
+	for i := 0; i < len(urls); i += batchSize {
+		end := i + batchSize
+		if end > len(urls) {
+			end = len(urls)
+		}
+		batches = append(batches, urls[i:end])
+	}
+
+	return batches
+}
+
+// BatchResult 批量提交结果
+type BatchResult struct {
+	TotalURLs    int
+	SuccessCount int
+	FailCount    int
+	Errors       []string
+	SuccessURLs  []string
+	StatusCodes  []int // 各引擎 HTTP 状态码（按 SearchEngines 顺序，含重试后最后一次）
+}
+
+// String 结果字符串表示
+func (r *BatchResult) String() string {
+	return fmt.Sprintf("提交完成: 总数=%d, 成功=%d, 失败=%d, 错误数=%d, statusCodes=%v",
+		r.TotalURLs, r.SuccessCount, r.FailCount, len(r.Errors), r.StatusCodes)
+}
@@ -0,0 +1,44 @@
+// Package sitemapdirty 在 web 与 app 之间传递「站点内容已变更」的标记。
+// web 进程负责文章的发布/更新/删除，而 sitemap 文件由 app 进程生成并对外提供，
+// 两者独立部署且不互相调用，因此借数据库中的一条记录做跨进程通知。
+package sitemapdirty
+
+import (
+	"context"
+	"time"
+	"videoweb/pub/common/log"
+	"videoweb/pub/models/syncfilemod"
+
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+func recordFilter() bson.M {
+	return bson.M{syncfilemod.FN.SyncRecord.Job: syncfilemod.SitemapDirtyJob}
+}
+
+// Mark 标记内容已变更，app 侧会在下一个检查周期重新生成 sitemap
+func Mark(ctx context.Context) {
+	now := time.Now()
+	update := bson.M{
+		"$set": bson.M{
+			syncfilemod.FN.SyncRecord.RecordTime: now,
+			syncfilemod.FN.SyncRecord.UpdatedAt:  now,
+		},
+		"$setOnInsert": bson.M{
+			syncfilemod.FN.SyncRecord.Job:       syncfilemod.SitemapDirtyJob,
+			syncfilemod.FN.SyncRecord.CreatedAt: now,
+		},
+	}
+	if serr := syncfilemod.UpsertOneSyncRecord(ctx, recordFilter(), update); serr != nil {
+		log.Warn("sitemapdirty: mark failed", log.E(serr))
+	}
+}
+
+// LastDirtyAt 读取最近一次内容变更时间，无记录时返回零值
+func LastDirtyAt(ctx context.Context) time.Time {
+	record, serr := syncfilemod.FindOneSyncRecord(ctx, recordFilter())
+	if serr != nil || record == nil {
+		return time.Time{}
+	}
+	return record.RecordTime
+}