Initial commit: web_hghl_server
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
@@ -0,0 +1,124 @@
|
||||
package reportdata
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/dataReport"
|
||||
"videoweb/pub/models/eventmod"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
var isRuining = false
|
||||
|
||||
const (
|
||||
maxRetryCount = 10 // 最大重试次数
|
||||
maxReportNum = 100 // 最大上报数
|
||||
)
|
||||
|
||||
func Report() {
|
||||
//防止重入
|
||||
if isRuining {
|
||||
return
|
||||
}
|
||||
isRuining = true
|
||||
defer func() {
|
||||
isRuining = false
|
||||
}()
|
||||
ctx := context.Background()
|
||||
opts := options.Find()
|
||||
opts = opts.SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(1000)
|
||||
// 查询出一千条记录批量上传
|
||||
list, stdErr := eventmod.FindManyEvent(ctx, bson.M{
|
||||
"$or": []bson.M{
|
||||
bson.M{
|
||||
"status": eventmod.Default,
|
||||
},
|
||||
bson.M{
|
||||
"status": eventmod.Failure,
|
||||
"retryCount": bson.M{"$lte": maxRetryCount},
|
||||
},
|
||||
},
|
||||
}, opts)
|
||||
if stdErr != nil {
|
||||
log.Error("Get event list error", log.E(stdErr))
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
var eventList []*dataReport.CommonField
|
||||
for _, v := range list {
|
||||
event := &dataReport.CommonField{}
|
||||
err := json.Unmarshal([]byte(v.Data), &event)
|
||||
if err != nil {
|
||||
log.Error("event report json.Unmarshal fail", log.Any("eventData", v.Data), log.E(err))
|
||||
continue
|
||||
}
|
||||
eventList = append(eventList, event)
|
||||
if len(eventList) < maxReportNum {
|
||||
continue
|
||||
}
|
||||
// 批量上报
|
||||
err = dataReport.BatchReport(ctx, eventList)
|
||||
if err != nil {
|
||||
log.Error("Batch report error", log.E(err))
|
||||
// 将这些设置为失败
|
||||
onReportEventFail(ctx, eventList)
|
||||
} else {
|
||||
onReportEventSuccess(ctx, eventList)
|
||||
}
|
||||
eventList = []*dataReport.CommonField{} // 重新变成0个
|
||||
}
|
||||
|
||||
// 将剩余的批量上报
|
||||
err := dataReport.BatchReport(ctx, eventList)
|
||||
if err != nil {
|
||||
log.Error("Batch report error", log.E(err))
|
||||
// 将这些设置为失败
|
||||
onReportEventFail(ctx, eventList)
|
||||
} else {
|
||||
onReportEventSuccess(ctx, eventList)
|
||||
}
|
||||
}
|
||||
|
||||
func onReportEventSuccess(ctx context.Context, eventList []*dataReport.CommonField) {
|
||||
eventIds := []string{}
|
||||
for _, v := range eventList {
|
||||
eventIds = append(eventIds, v.EventID)
|
||||
}
|
||||
if len(eventIds) == 0 {
|
||||
return
|
||||
}
|
||||
err := eventmod.UpdateManyEvent(ctx, bson.M{
|
||||
"eventId": bson.M{"$in": eventIds},
|
||||
}, bson.M{
|
||||
"$set": bson.M{"status": eventmod.Success},
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("Update event fail", log.E(err))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func onReportEventFail(ctx context.Context, eventList []*dataReport.CommonField) {
|
||||
eventIds := []string{}
|
||||
for _, v := range eventList {
|
||||
eventIds = append(eventIds, v.EventID)
|
||||
}
|
||||
if len(eventIds) == 0 {
|
||||
return
|
||||
}
|
||||
err := eventmod.UpdateManyEvent(ctx, bson.M{
|
||||
"eventId": bson.M{"$in": eventIds},
|
||||
}, bson.M{
|
||||
"$set": bson.M{"status": eventmod.Failure},
|
||||
"$inc": bson.M{"retryCount": 1},
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("Update event fail", log.E(err))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
// Package seopush 为搜索引擎推送做兜底：补推已上架内容，并重试失败的任务。
|
||||
package seopush
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/models/playletmod"
|
||||
"videoweb/pub/models/seopushmod"
|
||||
"videoweb/pub/models/videoinformationmod"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/web/service/srv_seopush"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const (
|
||||
backfillLimit = 100
|
||||
retryLimit = 50
|
||||
taskKeepDays = 30
|
||||
)
|
||||
|
||||
var jobMu sync.Mutex
|
||||
|
||||
// Run 定时兜底任务
|
||||
func Run() {
|
||||
if !jobMu.TryLock() {
|
||||
log.Warn("seopush: skip job, previous run still in progress")
|
||||
return
|
||||
}
|
||||
defer jobMu.Unlock()
|
||||
|
||||
ctx := context.Background()
|
||||
if !srv_seopush.Enabled() {
|
||||
discardPendingTasks(ctx)
|
||||
return
|
||||
}
|
||||
webAddr := srv_seopush.SEOBaseURL(ctx)
|
||||
if webAddr == "" {
|
||||
return
|
||||
}
|
||||
backfillVideos(ctx)
|
||||
backfillPlaylets(ctx)
|
||||
retryPendingTasks(ctx, webAddr)
|
||||
cleanupExpiredTasks(ctx)
|
||||
}
|
||||
|
||||
func discardPendingTasks(ctx context.Context) {
|
||||
filter := bson.M{"$or": []bson.M{
|
||||
{seopushmod.FN.SeoPushTask.GoogleStatus: seopushmod.PushStatusPending},
|
||||
{seopushmod.FN.SeoPushTask.IndexNowStatus: seopushmod.PushStatusPending},
|
||||
}}
|
||||
list, serr := seopushmod.FindManySeoPushTasks(ctx, filter)
|
||||
if serr != nil {
|
||||
log.Warn("seopush: find pending tasks to discard failed", log.E(serr))
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
byKind := map[string][]int64{}
|
||||
seen := map[string]map[int64]struct{}{}
|
||||
for _, task := range list {
|
||||
if task == nil || task.VideoId <= 0 {
|
||||
continue
|
||||
}
|
||||
kind := kindFromTaskURL(task.Url)
|
||||
if kind == "" {
|
||||
continue
|
||||
}
|
||||
if seen[kind] == nil {
|
||||
seen[kind] = make(map[int64]struct{})
|
||||
}
|
||||
if _, ok := seen[kind][task.VideoId]; ok {
|
||||
continue
|
||||
}
|
||||
seen[kind][task.VideoId] = struct{}{}
|
||||
byKind[kind] = append(byKind[kind], task.VideoId)
|
||||
}
|
||||
count, serr := seopushmod.DelManySeoPushTask(ctx, filter)
|
||||
if serr != nil {
|
||||
log.Warn("seopush: discard pending tasks failed", log.E(serr))
|
||||
return
|
||||
}
|
||||
resetIndexNow(ctx, srv_seopush.KindVideo, byKind[srv_seopush.KindVideo])
|
||||
resetIndexNow(ctx, srv_seopush.KindPlaylet, byKind[srv_seopush.KindPlaylet])
|
||||
log.Info("seopush=skip=disabled", log.Any("discarded", count))
|
||||
}
|
||||
|
||||
func kindFromTaskURL(pageURL string) string {
|
||||
switch {
|
||||
case strings.Contains(pageURL, "/video-detail"):
|
||||
return srv_seopush.KindVideo
|
||||
case strings.Contains(pageURL, "/drama-detail"):
|
||||
return srv_seopush.KindPlaylet
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func resetIndexNow(ctx context.Context, kind string, ids []int64) {
|
||||
if len(ids) == 0 {
|
||||
return
|
||||
}
|
||||
in := bson.M{"$in": ids}
|
||||
setZero := bson.M{"$set": bson.M{}}
|
||||
var serr error
|
||||
switch kind {
|
||||
case srv_seopush.KindVideo:
|
||||
setZero["$set"] = bson.M{videoinformationmod.FN.VideoInformation.IndexNow: int64(0)}
|
||||
serr = videoinformationmod.UpdateManyVideoInformation(ctx,
|
||||
bson.M{videoinformationmod.FN.VideoInformation.Id: in}, setZero)
|
||||
case srv_seopush.KindPlaylet:
|
||||
setZero["$set"] = bson.M{playletmod.FN.Playlet.IndexNow: int64(0)}
|
||||
serr = playletmod.UpdateManyPlaylet(ctx,
|
||||
bson.M{playletmod.FN.Playlet.Id: in}, setZero)
|
||||
}
|
||||
if serr != nil {
|
||||
log.Warn("seopush: reset indexNow failed", log.Any("kind", kind), log.E(serr))
|
||||
}
|
||||
}
|
||||
|
||||
func indexNowPendingFilter(statusField, indexNowField string, statusVal interface{}) bson.M {
|
||||
return bson.M{
|
||||
statusField: statusVal,
|
||||
"$or": []bson.M{
|
||||
{indexNowField: 0},
|
||||
{indexNowField: bson.M{"$exists": false}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func backfillVideos(ctx context.Context) {
|
||||
opt := options.Find().
|
||||
SetSort(bson.M{videoinformationmod.FN.VideoInformation.UpdatedAt: -1}).
|
||||
SetLimit(backfillLimit)
|
||||
list, serr := videoinformationmod.FindManyVideoInformations(ctx, indexNowPendingFilter(
|
||||
videoinformationmod.FN.VideoInformation.Status,
|
||||
videoinformationmod.FN.VideoInformation.IndexNow,
|
||||
pubdef.InformationStatusVerifyOK,
|
||||
), opt)
|
||||
if serr != nil {
|
||||
log.Warn("seopush: backfill find videos failed", log.E(serr))
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
for _, item := range list {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
srv_seopush.RegisterUpdated(ctx, srv_seopush.KindVideo, item.Id, srv_seopush.PageVisibleInfo(item.Status))
|
||||
}
|
||||
log.Info("seopush: backfill videos registered", log.Any("count", len(list)))
|
||||
}
|
||||
|
||||
func backfillPlaylets(ctx context.Context) {
|
||||
opt := options.Find().
|
||||
SetSort(bson.M{playletmod.FN.Playlet.OnlineAt: -1}).
|
||||
SetLimit(backfillLimit)
|
||||
list, serr := playletmod.FindManyPlaylet(ctx, indexNowPendingFilter(
|
||||
playletmod.FN.Playlet.Status,
|
||||
playletmod.FN.Playlet.IndexNow,
|
||||
playletmod.StatusUp,
|
||||
), opt)
|
||||
if serr != nil {
|
||||
log.Warn("seopush: backfill find playlets failed", log.E(serr))
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
for _, item := range list {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
srv_seopush.RegisterUpdated(ctx, srv_seopush.KindPlaylet, item.Id,
|
||||
srv_seopush.PageVisiblePlaylet(item.Status, item.OnlineAt))
|
||||
}
|
||||
log.Info("seopush: backfill playlets registered", log.Any("count", len(list)))
|
||||
}
|
||||
|
||||
func retryPendingTasks(ctx context.Context, webAddr string) {
|
||||
conds := []bson.M{
|
||||
{
|
||||
seopushmod.FN.SeoPushTask.GoogleStatus: seopushmod.PushStatusPending,
|
||||
seopushmod.FN.SeoPushTask.GoogleRetry: bson.M{"$lt": seopushmod.MaxPushRetry},
|
||||
},
|
||||
{
|
||||
seopushmod.FN.SeoPushTask.IndexNowStatus: seopushmod.PushStatusPending,
|
||||
seopushmod.FN.SeoPushTask.IndexNowRetry: bson.M{"$lt": seopushmod.MaxPushRetry},
|
||||
},
|
||||
}
|
||||
opt := options.Find().
|
||||
SetSort(bson.M{seopushmod.FN.SeoPushTask.CreatedAt: 1}).
|
||||
SetLimit(retryLimit)
|
||||
taskList, serr := seopushmod.FindManySeoPushTasks(ctx, bson.M{"$or": conds}, opt)
|
||||
if serr != nil {
|
||||
log.Warn("seopush: find pending tasks failed", log.E(serr))
|
||||
return
|
||||
}
|
||||
if len(taskList) == 0 {
|
||||
return
|
||||
}
|
||||
for _, task := range taskList {
|
||||
if task == nil || strings.Contains(task.Url, "?") {
|
||||
continue
|
||||
}
|
||||
srv_seopush.PushTask(ctx, webAddr, task)
|
||||
}
|
||||
log.Info("seopush: pending tasks processed", log.Any("count", len(taskList)))
|
||||
}
|
||||
|
||||
func cleanupExpiredTasks(ctx context.Context) {
|
||||
deadline := time.Now().AddDate(0, 0, -taskKeepDays)
|
||||
count, serr := seopushmod.DelManySeoPushTask(ctx, bson.M{
|
||||
seopushmod.FN.SeoPushTask.UpdatedAt: bson.M{"$lt": deadline},
|
||||
})
|
||||
if serr != nil {
|
||||
log.Warn("seopush: cleanup expired tasks failed", log.E(serr))
|
||||
return
|
||||
}
|
||||
if count > 0 {
|
||||
log.Info("seopush: expired tasks cleaned", log.Any("count", count))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"videoweb/pub/common/laosiji"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/models/videoinformationmod"
|
||||
"videoweb/web/service/srv_videoinfo"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const transcodeBatchSize = 5
|
||||
const transcodeMaxFailCount = 3
|
||||
|
||||
var transcodeRunning int32
|
||||
var transcodeCheckRunning int32
|
||||
|
||||
const transcodeCheckBatchSize = 100
|
||||
|
||||
// TranscodeVideoByWatchTimes 按观看次数挑选尚未转码成功的视频，批量提交 H265 转码。
|
||||
func TranscodeVideoByWatchTimes() {
|
||||
if !atomic.CompareAndSwapInt32(&transcodeRunning, 0, 1) {
|
||||
log.Info("TranscodeVideoByWatchTimes skip: previous run still running")
|
||||
return
|
||||
}
|
||||
defer atomic.StoreInt32(&transcodeRunning, 0)
|
||||
|
||||
ctx := context.Background()
|
||||
if laosiji.LaosijiClient == nil || laosiji.LaosijiDomains == nil {
|
||||
log.Warn("TranscodeVideoByWatchTimes skip: laosiji not ready")
|
||||
return
|
||||
}
|
||||
if _, err := laosiji.LaosijiDomains.Get(ctx); err != nil {
|
||||
log.Warn("TranscodeVideoByWatchTimes skip: domains not ready", log.E(err))
|
||||
return
|
||||
}
|
||||
|
||||
filter := bson.M{
|
||||
videoinformationmod.FN.VideoInformation.HevcStatus: bson.M{
|
||||
"$in": bson.A{videoinformationmod.HevcStatusNone, videoinformationmod.HevcStatusFailed},
|
||||
},
|
||||
videoinformationmod.FN.VideoInformation.HevcFailCount: bson.M{"$lt": transcodeMaxFailCount},
|
||||
videoinformationmod.FN.VideoInformation.Video.Url: bson.M{"$ne": ""},
|
||||
}
|
||||
opts := options.Find().
|
||||
SetSort(bson.D{{Key: videoinformationmod.FN.VideoInformation.Real.WatchTimes, Value: -1}}).
|
||||
SetLimit(transcodeBatchSize)
|
||||
|
||||
videoList, serr := videoinformationmod.FindManyVideoInformations(ctx, filter, opts)
|
||||
if serr != nil {
|
||||
log.Error("TranscodeVideoByWatchTimes find video failed", log.Any("serr", serr))
|
||||
return
|
||||
}
|
||||
if len(videoList) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("TranscodeVideoByWatchTimes start", log.Any("count", len(videoList)))
|
||||
for _, v := range videoList {
|
||||
src := strings.TrimSpace(v.Video.Url)
|
||||
if src == "" {
|
||||
continue
|
||||
}
|
||||
if err := srv_videoinfo.SubmitTranscodeVideo(ctx, v.Id, src, v.Video.FileId); err != nil {
|
||||
log.Warn("TranscodeVideoByWatchTimes submit failed",
|
||||
log.Any("videoId", v.Id), log.Any("watchTimes", v.Real.WatchTimes), log.E(err))
|
||||
}
|
||||
}
|
||||
log.Info("TranscodeVideoByWatchTimes done", log.Any("count", len(videoList)))
|
||||
}
|
||||
|
||||
// CheckTranscodingResults 查询转码中任务并回写结果。
|
||||
func CheckTranscodingResults() {
|
||||
if !atomic.CompareAndSwapInt32(&transcodeCheckRunning, 0, 1) {
|
||||
log.Info("CheckTranscodingResults skip: previous run still running")
|
||||
return
|
||||
}
|
||||
defer atomic.StoreInt32(&transcodeCheckRunning, 0)
|
||||
|
||||
ctx := context.Background()
|
||||
if laosiji.LaosijiClient == nil || laosiji.LaosijiDomains == nil {
|
||||
return
|
||||
}
|
||||
|
||||
filter := bson.M{videoinformationmod.FN.VideoInformation.HevcStatus: videoinformationmod.HevcStatusPending}
|
||||
opts := options.Find().SetLimit(transcodeCheckBatchSize)
|
||||
videoList, serr := videoinformationmod.FindManyVideoInformations(ctx, filter, opts)
|
||||
if serr != nil {
|
||||
log.Error("CheckTranscodingResults find video failed", log.Any("serr", serr))
|
||||
return
|
||||
}
|
||||
if len(videoList) == 0 {
|
||||
return
|
||||
}
|
||||
for _, v := range videoList {
|
||||
src := strings.TrimSpace(v.Video.Url)
|
||||
if src == "" {
|
||||
continue
|
||||
}
|
||||
if err := srv_videoinfo.RefreshTranscodeVideo(ctx, v.Id, src, v.Video.FileId); err != nil {
|
||||
log.Warn("CheckTranscodingResults refresh failed", log.Any("videoId", v.Id), log.E(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user