231 lines
6.2 KiB
Go
231 lines
6.2 KiB
Go
// Package seopush 为搜索引擎推送做兜底:补推已上架内容,并重试失败的任务。
|
|
package seopush
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
"videoweb/pub/common/log"
|
|
"videoweb/pub/models/playletmod"
|
|
"videoweb/pub/models/seopushmod"
|
|
"videoweb/pub/models/videoinformationmod"
|
|
"videoweb/pub/pubdef"
|
|
"videoweb/web/service/srv_seopush"
|
|
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
const (
|
|
backfillLimit = 100
|
|
retryLimit = 50
|
|
taskKeepDays = 30
|
|
)
|
|
|
|
var jobMu sync.Mutex
|
|
|
|
// Run 定时兜底任务
|
|
func Run() {
|
|
if !jobMu.TryLock() {
|
|
log.Warn("seopush: skip job, previous run still in progress")
|
|
return
|
|
}
|
|
defer jobMu.Unlock()
|
|
|
|
ctx := context.Background()
|
|
if !srv_seopush.Enabled() {
|
|
discardPendingTasks(ctx)
|
|
return
|
|
}
|
|
webAddr := srv_seopush.SEOBaseURL(ctx)
|
|
if webAddr == "" {
|
|
return
|
|
}
|
|
backfillVideos(ctx)
|
|
backfillPlaylets(ctx)
|
|
retryPendingTasks(ctx, webAddr)
|
|
cleanupExpiredTasks(ctx)
|
|
}
|
|
|
|
func discardPendingTasks(ctx context.Context) {
|
|
filter := bson.M{"$or": []bson.M{
|
|
{seopushmod.FN.SeoPushTask.GoogleStatus: seopushmod.PushStatusPending},
|
|
{seopushmod.FN.SeoPushTask.IndexNowStatus: seopushmod.PushStatusPending},
|
|
}}
|
|
list, serr := seopushmod.FindManySeoPushTasks(ctx, filter)
|
|
if serr != nil {
|
|
log.Warn("seopush: find pending tasks to discard failed", log.E(serr))
|
|
return
|
|
}
|
|
if len(list) == 0 {
|
|
return
|
|
}
|
|
byKind := map[string][]int64{}
|
|
seen := map[string]map[int64]struct{}{}
|
|
for _, task := range list {
|
|
if task == nil || task.VideoId <= 0 {
|
|
continue
|
|
}
|
|
kind := kindFromTaskURL(task.Url)
|
|
if kind == "" {
|
|
continue
|
|
}
|
|
if seen[kind] == nil {
|
|
seen[kind] = make(map[int64]struct{})
|
|
}
|
|
if _, ok := seen[kind][task.VideoId]; ok {
|
|
continue
|
|
}
|
|
seen[kind][task.VideoId] = struct{}{}
|
|
byKind[kind] = append(byKind[kind], task.VideoId)
|
|
}
|
|
count, serr := seopushmod.DelManySeoPushTask(ctx, filter)
|
|
if serr != nil {
|
|
log.Warn("seopush: discard pending tasks failed", log.E(serr))
|
|
return
|
|
}
|
|
resetIndexNow(ctx, srv_seopush.KindVideo, byKind[srv_seopush.KindVideo])
|
|
resetIndexNow(ctx, srv_seopush.KindPlaylet, byKind[srv_seopush.KindPlaylet])
|
|
log.Info("seopush=skip=disabled", log.Any("discarded", count))
|
|
}
|
|
|
|
func kindFromTaskURL(pageURL string) string {
|
|
switch {
|
|
case strings.Contains(pageURL, "/video-detail"):
|
|
return srv_seopush.KindVideo
|
|
case strings.Contains(pageURL, "/drama-detail"):
|
|
return srv_seopush.KindPlaylet
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func resetIndexNow(ctx context.Context, kind string, ids []int64) {
|
|
if len(ids) == 0 {
|
|
return
|
|
}
|
|
in := bson.M{"$in": ids}
|
|
setZero := bson.M{"$set": bson.M{}}
|
|
var serr error
|
|
switch kind {
|
|
case srv_seopush.KindVideo:
|
|
setZero["$set"] = bson.M{videoinformationmod.FN.VideoInformation.IndexNow: int64(0)}
|
|
serr = videoinformationmod.UpdateManyVideoInformation(ctx,
|
|
bson.M{videoinformationmod.FN.VideoInformation.Id: in}, setZero)
|
|
case srv_seopush.KindPlaylet:
|
|
setZero["$set"] = bson.M{playletmod.FN.Playlet.IndexNow: int64(0)}
|
|
serr = playletmod.UpdateManyPlaylet(ctx,
|
|
bson.M{playletmod.FN.Playlet.Id: in}, setZero)
|
|
}
|
|
if serr != nil {
|
|
log.Warn("seopush: reset indexNow failed", log.Any("kind", kind), log.E(serr))
|
|
}
|
|
}
|
|
|
|
func indexNowPendingFilter(statusField, indexNowField string, statusVal interface{}) bson.M {
|
|
return bson.M{
|
|
statusField: statusVal,
|
|
"$or": []bson.M{
|
|
{indexNowField: 0},
|
|
{indexNowField: bson.M{"$exists": false}},
|
|
},
|
|
}
|
|
}
|
|
|
|
func backfillVideos(ctx context.Context) {
|
|
opt := options.Find().
|
|
SetSort(bson.M{videoinformationmod.FN.VideoInformation.UpdatedAt: -1}).
|
|
SetLimit(backfillLimit)
|
|
list, serr := videoinformationmod.FindManyVideoInformations(ctx, indexNowPendingFilter(
|
|
videoinformationmod.FN.VideoInformation.Status,
|
|
videoinformationmod.FN.VideoInformation.IndexNow,
|
|
pubdef.InformationStatusVerifyOK,
|
|
), opt)
|
|
if serr != nil {
|
|
log.Warn("seopush: backfill find videos failed", log.E(serr))
|
|
return
|
|
}
|
|
if len(list) == 0 {
|
|
return
|
|
}
|
|
for _, item := range list {
|
|
if item == nil {
|
|
continue
|
|
}
|
|
srv_seopush.RegisterUpdated(ctx, srv_seopush.KindVideo, item.Id, srv_seopush.PageVisibleInfo(item.Status))
|
|
}
|
|
log.Info("seopush: backfill videos registered", log.Any("count", len(list)))
|
|
}
|
|
|
|
func backfillPlaylets(ctx context.Context) {
|
|
opt := options.Find().
|
|
SetSort(bson.M{playletmod.FN.Playlet.OnlineAt: -1}).
|
|
SetLimit(backfillLimit)
|
|
list, serr := playletmod.FindManyPlaylet(ctx, indexNowPendingFilter(
|
|
playletmod.FN.Playlet.Status,
|
|
playletmod.FN.Playlet.IndexNow,
|
|
playletmod.StatusUp,
|
|
), opt)
|
|
if serr != nil {
|
|
log.Warn("seopush: backfill find playlets failed", log.E(serr))
|
|
return
|
|
}
|
|
if len(list) == 0 {
|
|
return
|
|
}
|
|
for _, item := range list {
|
|
if item == nil {
|
|
continue
|
|
}
|
|
srv_seopush.RegisterUpdated(ctx, srv_seopush.KindPlaylet, item.Id,
|
|
srv_seopush.PageVisiblePlaylet(item.Status, item.OnlineAt))
|
|
}
|
|
log.Info("seopush: backfill playlets registered", log.Any("count", len(list)))
|
|
}
|
|
|
|
func retryPendingTasks(ctx context.Context, webAddr string) {
|
|
conds := []bson.M{
|
|
{
|
|
seopushmod.FN.SeoPushTask.GoogleStatus: seopushmod.PushStatusPending,
|
|
seopushmod.FN.SeoPushTask.GoogleRetry: bson.M{"$lt": seopushmod.MaxPushRetry},
|
|
},
|
|
{
|
|
seopushmod.FN.SeoPushTask.IndexNowStatus: seopushmod.PushStatusPending,
|
|
seopushmod.FN.SeoPushTask.IndexNowRetry: bson.M{"$lt": seopushmod.MaxPushRetry},
|
|
},
|
|
}
|
|
opt := options.Find().
|
|
SetSort(bson.M{seopushmod.FN.SeoPushTask.CreatedAt: 1}).
|
|
SetLimit(retryLimit)
|
|
taskList, serr := seopushmod.FindManySeoPushTasks(ctx, bson.M{"$or": conds}, opt)
|
|
if serr != nil {
|
|
log.Warn("seopush: find pending tasks failed", log.E(serr))
|
|
return
|
|
}
|
|
if len(taskList) == 0 {
|
|
return
|
|
}
|
|
for _, task := range taskList {
|
|
if task == nil || strings.Contains(task.Url, "?") {
|
|
continue
|
|
}
|
|
srv_seopush.PushTask(ctx, webAddr, task)
|
|
}
|
|
log.Info("seopush: pending tasks processed", log.Any("count", len(taskList)))
|
|
}
|
|
|
|
func cleanupExpiredTasks(ctx context.Context) {
|
|
deadline := time.Now().AddDate(0, 0, -taskKeepDays)
|
|
count, serr := seopushmod.DelManySeoPushTask(ctx, bson.M{
|
|
seopushmod.FN.SeoPushTask.UpdatedAt: bson.M{"$lt": deadline},
|
|
})
|
|
if serr != nil {
|
|
log.Warn("seopush: cleanup expired tasks failed", log.E(serr))
|
|
return
|
|
}
|
|
if count > 0 {
|
|
log.Info("seopush: expired tasks cleaned", log.Any("count", count))
|
|
}
|
|
}
|