// Package seopush 为搜索引擎推送做兜底:补推已上架内容,并重试失败的任务。 package seopush import ( "context" "strings" "sync" "time" "videoweb/pub/common/log" "videoweb/pub/models/playletmod" "videoweb/pub/models/seopushmod" "videoweb/pub/models/videoinformationmod" "videoweb/pub/pubdef" "videoweb/web/service/srv_seopush" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo/options" ) const ( backfillLimit = 100 retryLimit = 50 taskKeepDays = 30 ) var jobMu sync.Mutex // Run 定时兜底任务 func Run() { if !jobMu.TryLock() { log.Warn("seopush: skip job, previous run still in progress") return } defer jobMu.Unlock() ctx := context.Background() if !srv_seopush.Enabled() { discardPendingTasks(ctx) return } webAddr := srv_seopush.SEOBaseURL(ctx) if webAddr == "" { return } backfillVideos(ctx) backfillPlaylets(ctx) retryPendingTasks(ctx, webAddr) cleanupExpiredTasks(ctx) } func discardPendingTasks(ctx context.Context) { filter := bson.M{"$or": []bson.M{ {seopushmod.FN.SeoPushTask.GoogleStatus: seopushmod.PushStatusPending}, {seopushmod.FN.SeoPushTask.IndexNowStatus: seopushmod.PushStatusPending}, }} list, serr := seopushmod.FindManySeoPushTasks(ctx, filter) if serr != nil { log.Warn("seopush: find pending tasks to discard failed", log.E(serr)) return } if len(list) == 0 { return } byKind := map[string][]int64{} seen := map[string]map[int64]struct{}{} for _, task := range list { if task == nil || task.VideoId <= 0 { continue } kind := kindFromTaskURL(task.Url) if kind == "" { continue } if seen[kind] == nil { seen[kind] = make(map[int64]struct{}) } if _, ok := seen[kind][task.VideoId]; ok { continue } seen[kind][task.VideoId] = struct{}{} byKind[kind] = append(byKind[kind], task.VideoId) } count, serr := seopushmod.DelManySeoPushTask(ctx, filter) if serr != nil { log.Warn("seopush: discard pending tasks failed", log.E(serr)) return } resetIndexNow(ctx, srv_seopush.KindVideo, byKind[srv_seopush.KindVideo]) resetIndexNow(ctx, srv_seopush.KindPlaylet, byKind[srv_seopush.KindPlaylet]) log.Info("seopush=skip=disabled", log.Any("discarded", count)) } func kindFromTaskURL(pageURL string) string { switch { case strings.Contains(pageURL, "/video-detail"): return srv_seopush.KindVideo case strings.Contains(pageURL, "/drama-detail"): return srv_seopush.KindPlaylet default: return "" } } func resetIndexNow(ctx context.Context, kind string, ids []int64) { if len(ids) == 0 { return } in := bson.M{"$in": ids} setZero := bson.M{"$set": bson.M{}} var serr error switch kind { case srv_seopush.KindVideo: setZero["$set"] = bson.M{videoinformationmod.FN.VideoInformation.IndexNow: int64(0)} serr = videoinformationmod.UpdateManyVideoInformation(ctx, bson.M{videoinformationmod.FN.VideoInformation.Id: in}, setZero) case srv_seopush.KindPlaylet: setZero["$set"] = bson.M{playletmod.FN.Playlet.IndexNow: int64(0)} serr = playletmod.UpdateManyPlaylet(ctx, bson.M{playletmod.FN.Playlet.Id: in}, setZero) } if serr != nil { log.Warn("seopush: reset indexNow failed", log.Any("kind", kind), log.E(serr)) } } func indexNowPendingFilter(statusField, indexNowField string, statusVal interface{}) bson.M { return bson.M{ statusField: statusVal, "$or": []bson.M{ {indexNowField: 0}, {indexNowField: bson.M{"$exists": false}}, }, } } func backfillVideos(ctx context.Context) { opt := options.Find(). SetSort(bson.M{videoinformationmod.FN.VideoInformation.UpdatedAt: -1}). SetLimit(backfillLimit) list, serr := videoinformationmod.FindManyVideoInformations(ctx, indexNowPendingFilter( videoinformationmod.FN.VideoInformation.Status, videoinformationmod.FN.VideoInformation.IndexNow, pubdef.InformationStatusVerifyOK, ), opt) if serr != nil { log.Warn("seopush: backfill find videos failed", log.E(serr)) return } if len(list) == 0 { return } for _, item := range list { if item == nil { continue } srv_seopush.RegisterUpdated(ctx, srv_seopush.KindVideo, item.Id, srv_seopush.PageVisibleInfo(item.Status)) } log.Info("seopush: backfill videos registered", log.Any("count", len(list))) } func backfillPlaylets(ctx context.Context) { opt := options.Find(). SetSort(bson.M{playletmod.FN.Playlet.OnlineAt: -1}). SetLimit(backfillLimit) list, serr := playletmod.FindManyPlaylet(ctx, indexNowPendingFilter( playletmod.FN.Playlet.Status, playletmod.FN.Playlet.IndexNow, playletmod.StatusUp, ), opt) if serr != nil { log.Warn("seopush: backfill find playlets failed", log.E(serr)) return } if len(list) == 0 { return } for _, item := range list { if item == nil { continue } srv_seopush.RegisterUpdated(ctx, srv_seopush.KindPlaylet, item.Id, srv_seopush.PageVisiblePlaylet(item.Status, item.OnlineAt)) } log.Info("seopush: backfill playlets registered", log.Any("count", len(list))) } func retryPendingTasks(ctx context.Context, webAddr string) { conds := []bson.M{ { seopushmod.FN.SeoPushTask.GoogleStatus: seopushmod.PushStatusPending, seopushmod.FN.SeoPushTask.GoogleRetry: bson.M{"$lt": seopushmod.MaxPushRetry}, }, { seopushmod.FN.SeoPushTask.IndexNowStatus: seopushmod.PushStatusPending, seopushmod.FN.SeoPushTask.IndexNowRetry: bson.M{"$lt": seopushmod.MaxPushRetry}, }, } opt := options.Find(). SetSort(bson.M{seopushmod.FN.SeoPushTask.CreatedAt: 1}). SetLimit(retryLimit) taskList, serr := seopushmod.FindManySeoPushTasks(ctx, bson.M{"$or": conds}, opt) if serr != nil { log.Warn("seopush: find pending tasks failed", log.E(serr)) return } if len(taskList) == 0 { return } for _, task := range taskList { if task == nil || strings.Contains(task.Url, "?") { continue } srv_seopush.PushTask(ctx, webAddr, task) } log.Info("seopush: pending tasks processed", log.Any("count", len(taskList))) } func cleanupExpiredTasks(ctx context.Context) { deadline := time.Now().AddDate(0, 0, -taskKeepDays) count, serr := seopushmod.DelManySeoPushTask(ctx, bson.M{ seopushmod.FN.SeoPushTask.UpdatedAt: bson.M{"$lt": deadline}, }) if serr != nil { log.Warn("seopush: cleanup expired tasks failed", log.E(serr)) return } if count > 0 { log.Info("seopush: expired tasks cleaned", log.Any("count", count)) } }