package esvideo import ( "fmt" "time" "91porn-server/common/elastic" "91porn-server/common/log" "91porn-server/common/timeutil/timerange" "91porn-server/models/s/statrecordmod" "91porn-server/models/v/vidmod" "91porn-server/skd/skdg" ) var es *elastic.Client func RunSync(defaultDate time.Time) { es = skdg.VideoES log.Info("Elastic Search video sync start...") defer log.Info("Elastic Search video sync end.") // 获取上次数据同步的记录时间 lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncVideoJob, defaultDate) if err != nil { log.Error("Elastic sync video data RunSync get last record time failed", log.E(err)) return } var now = time.Now() var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) for recentMinute.After(lastSyncAt) { var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute) if end.After(recentMinute) { end = recentMinute } var subTimeRange = timerange.TimeRange{ Head: lastSyncAt, Tail: end, } var page, size = 1, 1000 hasNext := true var data []*vidmod.VideoModel for hasNext { data, hasNext, err = vidmod.GetVideoListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size) if err != nil { log.Error("Elastic sync video data GetVideoListByUpdateTimeRange err! ", log.E(err)) return } if hasNext && len(data) > 0 { _ = SyncDataToES(data) page++ } else { if err = commit(data, subTimeRange.Tail); err != nil { log.Error("Elastic sync video data RunSync commit err! ", log.E(err)) return } } } lastSyncAt = end } } func commit(data []*vidmod.VideoModel, syncTime time.Time) error { if len(data) <= 0 { _ = statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncVideoJob, syncTime) return nil } if err := SyncDataToES(data); err != nil { return fmt.Errorf("Elastic sync commit video data SyncDataToES failed!, err:%+v\n", err) } if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncVideoJob, syncTime); err != nil { return fmt.Errorf("Elastic sync commit video data UpsertOneTrans failed!, err:%+v\n", err) } return nil }