package esmedia import ( "fmt" "time" "91porn-server/common/elastic" "91porn-server/common/log" "91porn-server/models/s/statrecordmod" "91porn-server/models/v/mediamod" "91porn-server/skd/skdg" "go.mongodb.org/mongo-driver/bson/primitive" ) var es *elastic.Client func RunSync(defaultDate time.Time) { es = skdg.VideoES log.Info("Elastic Search ACG media sync start...") defer log.Info("Elastic Search ACG media sync end.") // MongoDB 日期精度为毫秒;本轮开始后的修改必须留给下一轮再次同步。 startedAt := time.Now().Truncate(time.Millisecond) lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncMediaJob, defaultDate) if err != nil { log.Error("Elastic sync ACG media get last record time failed", log.E(err)) return } if !startedAt.After(lastSyncAt) { return } // 固定 ID 上界,避免持续新增数据使本轮无法结束。 maxID, err := mediamod.GetLatestIDForESSync() if err != nil { log.Error("Elastic sync ACG media get latest ID failed", log.E(err)) return } sync := mediaSync{ read: mediamod.GetListForESSync, write: SyncDataToES, checkpoint: func(at time.Time) error { return statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncMediaJob, at) }, } if err = sync.run(lastSyncAt, startedAt, maxID); err != nil { log.Error("Elastic sync ACG media failed; checkpoint not confirmed", log.E(err)) } } type mediaSync struct { read func(time.Time, primitive.ObjectID, primitive.ObjectID, int) ([]*mediamod.Media, error) write func([]*mediamod.Media) error checkpoint func(time.Time) error } func (s mediaSync) run(since, startedAt time.Time, maxID primitive.ObjectID) error { const size = 1000 var after primitive.ObjectID for !maxID.IsZero() { data, err := s.read(since, after, maxID, size) if err != nil { return fmt.Errorf("read media batch: %w", err) } if len(data) == 0 { break } if err = s.write(data); err != nil { return fmt.Errorf("write media batch: %w", err) } after = data[len(data)-1].ID if len(data) < size || after == maxID { break } } // 所有批次及每条 ES 写入均成功后才推进;失败时下一轮从原进度幂等重试。 if err := s.checkpoint(startedAt); err != nil { return fmt.Errorf("save media checkpoint: %w", err) } return nil }