Initial commit

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

skd/job/elastic/esmedia/sync.go

@@ -0,0 +1,80 @@
+package esmedia
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/v/mediamod"
+	"91porn-server/skd/skdg"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+var es *elastic.Client
+
+func RunSync(defaultDate time.Time) {
+	es = skdg.VideoES
+	log.Info("Elastic Search ACG media sync start...")
+	defer log.Info("Elastic Search ACG media sync end.")
+	// MongoDB 日期精度为毫秒；本轮开始后的修改必须留给下一轮再次同步。
+	startedAt := time.Now().Truncate(time.Millisecond)
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncMediaJob, defaultDate)
+	if err != nil {
+		log.Error("Elastic sync ACG media get last record time failed", log.E(err))
+		return
+	}
+	if !startedAt.After(lastSyncAt) {
+		return
+	}
+	// 固定 ID 上界，避免持续新增数据使本轮无法结束。
+	maxID, err := mediamod.GetLatestIDForESSync()
+	if err != nil {
+		log.Error("Elastic sync ACG media get latest ID failed", log.E(err))
+		return
+	}
+	sync := mediaSync{
+		read:  mediamod.GetListForESSync,
+		write: SyncDataToES,
+		checkpoint: func(at time.Time) error {
+			return statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncMediaJob, at)
+		},
+	}
+	if err = sync.run(lastSyncAt, startedAt, maxID); err != nil {
+		log.Error("Elastic sync ACG media failed; checkpoint not confirmed", log.E(err))
+	}
+}
+
+type mediaSync struct {
+	read       func(time.Time, primitive.ObjectID, primitive.ObjectID, int) ([]*mediamod.Media, error)
+	write      func([]*mediamod.Media) error
+	checkpoint func(time.Time) error
+}
+
+func (s mediaSync) run(since, startedAt time.Time, maxID primitive.ObjectID) error {
+	const size = 1000
+	var after primitive.ObjectID
+	for !maxID.IsZero() {
+		data, err := s.read(since, after, maxID, size)
+		if err != nil {
+			return fmt.Errorf("read media batch: %w", err)
+		}
+		if len(data) == 0 {
+			break
+		}
+		if err = s.write(data); err != nil {
+			return fmt.Errorf("write media batch: %w", err)
+		}
+		after = data[len(data)-1].ID
+		if len(data) < size || after == maxID {
+			break
+		}
+	}
+	// 所有批次及每条 ES 写入均成功后才推进；失败时下一轮从原进度幂等重试。
+	if err := s.checkpoint(startedAt); err != nil {
+		return fmt.Errorf("save media checkpoint: %w", err)
+	}
+	return nil
+}