init: web_hgdjc_server
This commit is contained in:
@@ -0,0 +1,305 @@
|
||||
package googleanalytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
"videoweb/pub/common/httputil"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/srvdef"
|
||||
"videoweb/pub/common/utils"
|
||||
"videoweb/pub/models/analyticsdailystat"
|
||||
"videoweb/pub/models/analyticsstat"
|
||||
"videoweb/web/webg"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
analyticsdata "google.golang.org/api/analyticsdata/v1beta"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
// HourlyData 用于存储和排序每小时的数据
|
||||
type HourlyData struct {
|
||||
Hour int
|
||||
Users int64
|
||||
}
|
||||
type ChannelAnalyzeReq struct {
|
||||
Channel string `from:"channel" json:"channel" binding:"required"` // 渠道
|
||||
Begin time.Time `json:"begin" form:"begin"` // 开始时间
|
||||
End time.Time `json:"end" form:"end"` // 结束时间
|
||||
AppId *int32 `from:"appId" json:"appId" ` // 应用id
|
||||
}
|
||||
|
||||
type AnalyzeItem struct {
|
||||
Channel string `json:"channel" form:"channel"` // 渠道
|
||||
AppId int32 `json:"appId" form:"appId"` // 应用ID
|
||||
AppName string `json:"appName" form:"appName"` // 应用名称
|
||||
Begin time.Time `json:"begin" form:"begin"` // 开始时间
|
||||
End time.Time `json:"end" form:"end"` // 结束时间
|
||||
RegisterCount int64 `json:"registerCount" form:"registerCount"` // 总注册IP数
|
||||
RegisterIpCount int64 `json:"registerIpCount" form:"registerIpCount"` // 总注册IP数
|
||||
VisitCount int64 `json:"visitCount" form:"visitCount"` // 总访问数
|
||||
AdvertiseClick int64 `json:"advertiseClick" form:"advertiseClick"` // 广告点击数
|
||||
RechargeCount int64 `json:"rechargeCount" form:"rechargeCount"` // 充值次数
|
||||
RechargeAmount int64 `json:"rechargeAmount" form:"rechargeAmount"` // 充值金额
|
||||
RechargeUsers int64 `json:"rechargeUsers" form:"rechargeUsers"` // 充值用户数
|
||||
YesTodayRegisterCount int64 `json:"yesTodayRegisterCount" form:"yesTodayRegisterCount"` // 昨日注册IP数
|
||||
YesTodayVisitCount int64 `json:"yesTodayVisitCount" form:"yesTodayVisitCount"` // 昨日访问数
|
||||
YesTodayRechargeAmount int64 `json:"yesTodayRechargeAmount" form:"yesTodayRechargeAmount"` // 昨日充值
|
||||
YesTodayRetainRate int64 `json:"yesTodayRetainRate" form:"yesTodayRetainRate"` // 昨日次流
|
||||
}
|
||||
type ChannelAnalyzeResp struct {
|
||||
List []*AnalyzeItem `json:"list" form:"list"`
|
||||
}
|
||||
type AnalyzeResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data ChannelAnalyzeResp `json:"data"`
|
||||
}
|
||||
|
||||
func AnalyticsData(appId int64, appName string, propertyId string) {
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
// 1. 认证授权 - 使用服务账号的 JSON 密钥文件
|
||||
credentialsFile := "composed-anvil-470512-r8-fd674ac0d7dc.json"
|
||||
b, err := os.ReadFile(credentialsFile)
|
||||
if err != nil {
|
||||
log.Error("AnalyticsData 读取凭证文件失败", log.E(err))
|
||||
return
|
||||
}
|
||||
|
||||
curStatTime := utils.RecentMinute(now.Add(time.Minute), 30) // 按20分钟对齐
|
||||
|
||||
data := analyticsstat.AnalyticsInfo{
|
||||
AppId: appId,
|
||||
SumDate: curStatTime,
|
||||
AppName: appName,
|
||||
DaySumDate: utils.BeginOfTime(curStatTime),
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
// 2. 创建 API 客户端
|
||||
client, err := analyticsdata.NewService(ctx, option.WithCredentialsJSON(b))
|
||||
if err != nil {
|
||||
log.Error("AnalyticsData 无法创建 Analytics Data 客户端", log.E(err))
|
||||
return
|
||||
}
|
||||
if client == nil {
|
||||
log.Error("AnalyticsData client 为空")
|
||||
return
|
||||
}
|
||||
|
||||
req := &analyticsdata.RunRealtimeReportRequest{
|
||||
// 虽然实时报告默认就是最近30分钟，但你可以通过 dateRanges 进行限制（如果需要的话）
|
||||
// DateRanges: []*analyticsdata.DateRange{
|
||||
// {StartDate: "30 minutes ago", EndDate: "now"}, // 某些客户端库可能支持相对时间
|
||||
// },
|
||||
Dimensions: []*analyticsdata.Dimension{
|
||||
// 添加你关心的维度，例如国家或地区
|
||||
{Name: "country"},
|
||||
// 实时报告支持的维度有限，请参考官方文档：
|
||||
// https://developers.google.com/analytics/devguides/reporting/data/v1/realtime-basics?hl=zh-cn
|
||||
},
|
||||
Metrics: []*analyticsdata.Metric{
|
||||
{Name: "activeUsers"}, // 获取活跃用户数
|
||||
},
|
||||
}
|
||||
|
||||
// 4. 调用 RealTime API
|
||||
resp, err := client.Properties.RunRealtimeReport(propertyId, req).Do()
|
||||
if err != nil {
|
||||
log.Error("AnalyticsData RunRealtimeReport 请求报告出错", log.E(err))
|
||||
return
|
||||
}
|
||||
if resp == nil {
|
||||
log.Error("AnalyticsData RunRealtimeReport resp 为空")
|
||||
return
|
||||
}
|
||||
|
||||
for _, row := range resp.Rows {
|
||||
if row == nil || len(row.MetricValues) == 0 || row.MetricValues[0] == nil {
|
||||
continue
|
||||
}
|
||||
activeUsers, _ := strconv.ParseInt(row.MetricValues[0].Value, 10, 64)
|
||||
data.ActiveUsers += activeUsers
|
||||
}
|
||||
|
||||
yestodayReq := &analyticsdata.RunReportRequest{
|
||||
DateRanges: []*analyticsdata.DateRange{
|
||||
{StartDate: "yesterday", EndDate: "yesterday"}, // 查询昨天
|
||||
},
|
||||
Dimensions: []*analyticsdata.Dimension{
|
||||
{Name: "date"}, // 核心：添加日期维度，按天分组
|
||||
},
|
||||
Metrics: []*analyticsdata.Metric{
|
||||
{Name: "screenPageViews"},
|
||||
{Name: "totalUsers"},
|
||||
{Name: "activeUsers"}, // 获取活跃用户数
|
||||
{Name: "sessions"}, // 会话数
|
||||
{Name: "newUsers"}, // 新增数量
|
||||
},
|
||||
}
|
||||
|
||||
yestodayResp, err := client.Properties.RunReport(propertyId, yestodayReq).Do()
|
||||
if err != nil {
|
||||
log.Error("AnalyticsData RunReport 请求报告出错", log.E(err))
|
||||
return
|
||||
}
|
||||
if yestodayResp == nil {
|
||||
log.Error("AnalyticsData RunReport yestodayResp 为空")
|
||||
return
|
||||
}
|
||||
for _, row := range yestodayResp.Rows {
|
||||
if row == nil || len(row.MetricValues) < 5 {
|
||||
continue
|
||||
}
|
||||
data.YestodayPV, _ = strconv.ParseInt(row.MetricValues[0].Value, 10, 64)
|
||||
data.YestodayUV, _ = strconv.ParseInt(row.MetricValues[1].Value, 10, 64)
|
||||
data.YestodayActiveUsers, _ = strconv.ParseInt(row.MetricValues[2].Value, 10, 64)
|
||||
data.YestodaySessions, _ = strconv.ParseInt(row.MetricValues[3].Value, 10, 64)
|
||||
data.YestodayNewUsers, _ = strconv.ParseInt(row.MetricValues[4].Value, 10, 64)
|
||||
}
|
||||
|
||||
analyticsstat.InsertOneAnalyticsInfo(ctx, &data)
|
||||
}
|
||||
|
||||
func AnalyticsDailyData(appId int64, appName string, channel string, propertyId string) {
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
// 1. 认证授权 - 使用服务账号的 JSON 密钥文件
|
||||
credentialsFile := "composed-anvil-470512-r8-fd674ac0d7dc.json"
|
||||
b, err := os.ReadFile(credentialsFile)
|
||||
if err != nil {
|
||||
log.Error("AnalyticsDailyData 读取凭证文件失败", log.E(err))
|
||||
return
|
||||
}
|
||||
|
||||
// 2. 创建 API 客户端
|
||||
client, err := analyticsdata.NewService(ctx, option.WithCredentialsJSON(b))
|
||||
if err != nil {
|
||||
log.Error("AnalyticsDailyData 无法创建 Analytics Data 客户端", log.E(err))
|
||||
return
|
||||
}
|
||||
if client == nil {
|
||||
log.Error("AnalyticsDailyData client 为空")
|
||||
return
|
||||
}
|
||||
|
||||
data := analyticsdailystat.AnalyticsDailyInfo{
|
||||
AppId: appId,
|
||||
SumDate: utils.BeginOfTime(now),
|
||||
AppName: appName,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
yestodayReq := &analyticsdata.RunReportRequest{
|
||||
DateRanges: []*analyticsdata.DateRange{
|
||||
{StartDate: "7daysAgo", EndDate: "today"}, // 查询昨天
|
||||
},
|
||||
Dimensions: []*analyticsdata.Dimension{
|
||||
{Name: "date"}, // 核心：添加日期维度，按天分组
|
||||
},
|
||||
Metrics: []*analyticsdata.Metric{
|
||||
{Name: "screenPageViews"},
|
||||
{Name: "totalUsers"},
|
||||
{Name: "activeUsers"}, // 获取活跃用户数
|
||||
{Name: "sessions"}, // 会话数
|
||||
{Name: "newUsers"}, // 新增数量
|
||||
},
|
||||
}
|
||||
|
||||
yestodayResp, err := client.Properties.RunReport(propertyId, yestodayReq).Do()
|
||||
if err != nil {
|
||||
log.Error("AnalyticsDailyData RunReport 请求报告出错", log.E(err))
|
||||
return
|
||||
}
|
||||
if yestodayResp == nil {
|
||||
log.Error("AnalyticsDailyData RunReport yestodayResp 为空")
|
||||
return
|
||||
}
|
||||
for _, row := range yestodayResp.Rows {
|
||||
if row == nil || len(row.DimensionValues) == 0 || row.DimensionValues[0] == nil || len(row.MetricValues) < 5 {
|
||||
continue
|
||||
}
|
||||
// data.SumDate
|
||||
layout := "20060102" // Go 的特定格式：2006=年，01=月，02=日
|
||||
|
||||
// 解析字符串为时间对象
|
||||
dayTime, err := time.Parse(layout, row.DimensionValues[0].Value)
|
||||
if err != nil {
|
||||
log.Error("AnalyticsDailyData time.Parse 失败", log.E(err))
|
||||
continue
|
||||
}
|
||||
data.SumDate = dayTime.Add(-8 * time.Hour)
|
||||
data.YestodayPV, _ = strconv.ParseInt(row.MetricValues[0].Value, 10, 64)
|
||||
data.YestodayUV, _ = strconv.ParseInt(row.MetricValues[1].Value, 10, 64)
|
||||
data.YestodayActiveUsers, _ = strconv.ParseInt(row.MetricValues[2].Value, 10, 64)
|
||||
data.YestodaySessions, _ = strconv.ParseInt(row.MetricValues[3].Value, 10, 64)
|
||||
data.YestodayNewUsers, _ = strconv.ParseInt(row.MetricValues[4].Value, 10, 64)
|
||||
|
||||
LastFilter := bson.M{
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.SumDate: data.SumDate,
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.AppId: appId,
|
||||
}
|
||||
|
||||
_, serr := analyticsdailystat.FindOneAnalyticsDailyInfo(ctx, LastFilter)
|
||||
if serr != nil && serr.Code == srvdef.ErrNoData {
|
||||
analyticsdailystat.InsertOneAnalyticsDailyInfo(ctx, &data)
|
||||
} else {
|
||||
LastUpdate := bson.M{
|
||||
"$set": bson.M{
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.YestodayPV: data.YestodayPV,
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.YestodayUV: data.YestodayUV,
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.YestodayActiveUsers: data.YestodayActiveUsers,
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.YestodaySessions: data.YestodaySessions,
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.YestodayNewUsers: data.YestodayNewUsers,
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.UpdatedAt: now,
|
||||
},
|
||||
}
|
||||
analyticsdailystat.UpdateOneAnalyticsDailyInfo(ctx, LastFilter, LastUpdate)
|
||||
}
|
||||
}
|
||||
|
||||
analyzeList := ChannelAnalyzeResp{}
|
||||
req := ChannelAnalyzeReq{
|
||||
Channel: channel,
|
||||
Begin: utils.BeginOfTime(now.AddDate(0, 0, -1)),
|
||||
End: utils.EndOfTime(now.AddDate(0, 0, -1)),
|
||||
}
|
||||
|
||||
param, _ := utils.ToJsonM(req)
|
||||
innerStatUrl := webg.Conf.NewInnerStatUrl + "/api/stat/dailyStat/channel/analyze"
|
||||
resp := AnalyzeResp{}
|
||||
code, err := httputil.Client().POSTJsonWithJResp(&resp, innerStatUrl, nil, param)
|
||||
if err != nil {
|
||||
log.Error("ChannelAnalyze httputil innerStatUrl failed!", log.E(err))
|
||||
}
|
||||
if code != 200 {
|
||||
log.Error("ChannelAnalyze httputil innerStatUrl code1!=200", log.Any("code", code))
|
||||
}
|
||||
log.Info("ChannelAnalyze", log.Any("req", req), log.Any("innerStatUrl", innerStatUrl), log.Any("analyzeList", resp.Data))
|
||||
analyzeList = resp.Data
|
||||
|
||||
for _, channelAnalyze := range analyzeList.List {
|
||||
data.LeadNewCount += float64(channelAnalyze.RegisterCount)
|
||||
data.LeadAmount += float64(channelAnalyze.RechargeAmount)
|
||||
}
|
||||
LastFilter := bson.M{
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.SumDate: utils.BeginOfTime(now),
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.AppId: appId,
|
||||
}
|
||||
_, serr := analyticsdailystat.FindOneAnalyticsDailyInfo(ctx, LastFilter)
|
||||
if serr != nil && serr.Code == srvdef.ErrNoData {
|
||||
analyticsdailystat.InsertOneAnalyticsDailyInfo(ctx, &data)
|
||||
} else {
|
||||
LastUpdate := bson.M{
|
||||
"$set": bson.M{
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.LeadNewCount: data.LeadNewCount,
|
||||
analyticsdailystat.FN.AnalyticsDailyInfo.LeadAmount: data.LeadAmount,
|
||||
},
|
||||
}
|
||||
|
||||
analyticsdailystat.UpdateOneAnalyticsDailyInfo(ctx, LastFilter, LastUpdate)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
package llmstxt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/utils"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/web/service/srv_cfg"
|
||||
)
|
||||
|
||||
const (
|
||||
maxRecentToolsInBrief = 50 // llms.txt 中最新工具条数上限
|
||||
maxRecentArticlesBrief = 50 // llms.txt 中最新文章条数上限
|
||||
maxRecentInFull = 500 // llms-full.txt 中最新工具/文章条数上限
|
||||
)
|
||||
|
||||
var generateMu sync.Mutex
|
||||
|
||||
// GenerateLLMSTxt 生成 llms.txt（摘要）和 llms-full.txt（详细）
|
||||
func GenerateLLMSTxt() {
|
||||
generateMu.Lock()
|
||||
defer generateMu.Unlock()
|
||||
|
||||
ctx := context.Background()
|
||||
config, serr := srv_cfg.GetSystemConfig(ctx)
|
||||
if serr != nil || config == nil {
|
||||
log.Error("GenerateLLMSTxt GetSystemConfig", log.Any("err", serr))
|
||||
return
|
||||
}
|
||||
|
||||
//baseURL := strings.TrimSuffix(config.WebAddr, "/")
|
||||
|
||||
//categories, subCategories, cateNameMap, subCateNameMap := loadCategories(ctx)
|
||||
//recentTools := loadRecentTools(ctx, maxRecentToolsInBrief)
|
||||
//recentArticles := loadRecentArticles(ctx, maxRecentArticlesBrief)
|
||||
//
|
||||
//var brief strings.Builder
|
||||
//writeBrief(&brief, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld, recentTools, recentArticles)
|
||||
//
|
||||
//if err := writeTextFile("llms.txt", brief.String()); err != nil {
|
||||
// log.Error("GenerateLLMSTxt write llms.txt", log.Any("err", err))
|
||||
// return
|
||||
//}
|
||||
//if err := commitFile("llms.txt"); err != nil {
|
||||
// log.Error("GenerateLLMSTxt commit llms.txt", log.Any("err", err))
|
||||
// return
|
||||
//}
|
||||
//
|
||||
//allTools := loadRecentTools(ctx, maxRecentInFull)
|
||||
//allArticles := loadRecentArticles(ctx, maxRecentInFull)
|
||||
//
|
||||
//var full strings.Builder
|
||||
//writeFull(&full, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld, categories, subCategories, cateNameMap, subCateNameMap, allTools, allArticles)
|
||||
//
|
||||
//if err := writeTextFile("llms-full.txt", full.String()); err != nil {
|
||||
// log.Error("GenerateLLMSTxt write llms-full.txt", log.Any("err", err))
|
||||
// return
|
||||
//}
|
||||
//if err := commitFile("llms-full.txt"); err != nil {
|
||||
// log.Error("GenerateLLMSTxt commit llms-full.txt", log.Any("err", err))
|
||||
// return
|
||||
//}
|
||||
|
||||
//log.Info("GenerateLLMSTxt done",
|
||||
// log.Any("categories", len(categories)),
|
||||
// log.Any("subCategories", len(subCategories)),
|
||||
// log.Any("tools", len(allTools)),
|
||||
// log.Any("articles", len(allArticles)),
|
||||
//)
|
||||
}
|
||||
|
||||
// ── 数据加载 ──────────────────────────────────────────────────
|
||||
//
|
||||
//func loadCategories(ctx context.Context) (
|
||||
// categories toolcategorymod.ToolCategoryModelList,
|
||||
// subCategories toolsubcategorymod.ToolSubCategoryModelList,
|
||||
// cateNameMap map[int64]string,
|
||||
// subCateNameMap map[int64]string,
|
||||
//) {
|
||||
// cateFilter := bson.M{
|
||||
// toolcategorymod.FN.ToolCategory.Status: 1,
|
||||
// toolcategorymod.FN.ToolCategory.Deleted: bson.M{"$ne": 1},
|
||||
// }
|
||||
// cateOpt := options.Find().SetSort(bson.D{
|
||||
// {Key: toolcategorymod.FN.ToolCategory.Sort, Value: 1},
|
||||
// {Key: toolcategorymod.FN.ToolCategory.Id, Value: 1},
|
||||
// })
|
||||
// categories, _ = toolcategorymod.FindMany(ctx, cateFilter, cateOpt)
|
||||
//
|
||||
// subFilter := bson.M{
|
||||
// toolsubcategorymod.FN.ToolSubCategory.Status: 1,
|
||||
// toolsubcategorymod.FN.ToolSubCategory.Deleted: bson.M{"$ne": 1},
|
||||
// }
|
||||
// subOpt := options.Find().SetSort(bson.D{
|
||||
// {Key: toolsubcategorymod.FN.ToolSubCategory.Sort, Value: 1},
|
||||
// {Key: toolsubcategorymod.FN.ToolSubCategory.Id, Value: 1},
|
||||
// })
|
||||
// subCategories, _ = toolsubcategorymod.FindMany(ctx, subFilter, subOpt)
|
||||
//
|
||||
// cateNameMap = make(map[int64]string, len(categories))
|
||||
// for _, c := range categories {
|
||||
// cateNameMap[c.Id] = c.Name
|
||||
// }
|
||||
// subCateNameMap = make(map[int64]string, len(subCategories))
|
||||
// for _, s := range subCategories {
|
||||
// subCateNameMap[s.Id] = s.Name
|
||||
// }
|
||||
// return categories, subCategories, cateNameMap, subCateNameMap
|
||||
//}
|
||||
//
|
||||
//func loadRecentTools(ctx context.Context, limit int64) toolmod.ToolModelList {
|
||||
// opt := options.Find().
|
||||
// SetSort(bson.D{
|
||||
// {Key: toolmod.FN.Tool.Sort, Value: 1},
|
||||
// {Key: toolmod.FN.Tool.Hot, Value: -1},
|
||||
// {Key: toolmod.FN.Tool.CreatedAt, Value: -1},
|
||||
// }).
|
||||
// SetLimit(limit)
|
||||
// filter := bson.M{
|
||||
// toolmod.FN.Tool.ReviewStatus: 1,
|
||||
// toolmod.FN.Tool.Slug: bson.M{"$nin": []interface{}{"", nil}},
|
||||
// }
|
||||
// list, _ := toolmod.FindMany(ctx, filter, opt)
|
||||
// return list
|
||||
//}
|
||||
//
|
||||
//func loadRecentArticles(ctx context.Context, limit int64) articlemod.ArticleModelList {
|
||||
// opt := options.Find().
|
||||
// SetSort(bson.D{{Key: articlemod.FN.Article.CreatedAt, Value: -1}}).
|
||||
// SetLimit(limit)
|
||||
// filter := bson.M{
|
||||
// articlemod.FN.Article.ReviewStatus: 1,
|
||||
// articlemod.FN.Article.Slug: bson.M{"$nin": []interface{}{"", nil}},
|
||||
// }
|
||||
// list, _ := articlemod.FindMany(ctx, filter, opt)
|
||||
// return list
|
||||
//}
|
||||
//
|
||||
//// ── 内容构建 ──────────────────────────────────────────────────
|
||||
//
|
||||
//func writeBrief(
|
||||
// b *strings.Builder,
|
||||
// title, desc, baseURL string,
|
||||
// keywords []string,
|
||||
// recentTools toolmod.ToolModelList,
|
||||
// recentArticles articlemod.ArticleModelList,
|
||||
//) {
|
||||
// writeHeader(b, title, desc, keywords)
|
||||
// writeImportantPages(b, baseURL)
|
||||
//
|
||||
// if len(recentTools) > 0 {
|
||||
// fmt.Fprintf(b, "## 最新 AI 工具\n\n")
|
||||
// for _, t := range recentTools {
|
||||
// if t == nil || t.Slug == "" {
|
||||
// continue
|
||||
// }
|
||||
// summary := t.Intro
|
||||
// if summary == "" {
|
||||
// summary = t.Desc
|
||||
// }
|
||||
// if summary == "" {
|
||||
// summary = t.Title
|
||||
// }
|
||||
// fmt.Fprintf(b, "- [%s](%s): %s\n", t.Title, toolDetailURL(baseURL, t.Slug), truncate(summary, 100))
|
||||
// }
|
||||
// b.WriteString("\n")
|
||||
// }
|
||||
//
|
||||
// if len(recentArticles) > 0 {
|
||||
// fmt.Fprintf(b, "## 最新 AI 资讯\n\n")
|
||||
// for _, a := range recentArticles {
|
||||
// if a == nil || a.Slug == "" {
|
||||
// continue
|
||||
// }
|
||||
// summary := a.Intro
|
||||
// if summary == "" {
|
||||
// summary = a.Title
|
||||
// }
|
||||
// fmt.Fprintf(b, "- [%s](%s): %s\n", a.Title, articleDetailURL(baseURL, a.Slug), truncate(summary, 100))
|
||||
// }
|
||||
// b.WriteString("\n")
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//func GetCategorySlug(id int64, categories toolcategorymod.ToolCategoryModelList) string {
|
||||
// if id <= 0 {
|
||||
// return ""
|
||||
// }
|
||||
// if len(categories) == 0 {
|
||||
// return ""
|
||||
// }
|
||||
// for _, category := range categories {
|
||||
// if category.Id == id {
|
||||
// return category.Slug
|
||||
// }
|
||||
// }
|
||||
// return ""
|
||||
//}
|
||||
//func GetSubCategorySlug(id int64, categories toolsubcategorymod.ToolSubCategoryModelList) string {
|
||||
// if id <= 0 {
|
||||
// return ""
|
||||
// }
|
||||
// if len(categories) == 0 {
|
||||
// return ""
|
||||
// }
|
||||
// for _, category := range categories {
|
||||
// if category.Id == id {
|
||||
// return category.Slug
|
||||
// }
|
||||
// }
|
||||
// return ""
|
||||
//}
|
||||
//func writeFull(
|
||||
// b *strings.Builder,
|
||||
// title, desc, baseURL string,
|
||||
// keywords []string,
|
||||
// categories toolcategorymod.ToolCategoryModelList,
|
||||
// subCategories toolsubcategorymod.ToolSubCategoryModelList,
|
||||
// cateNameMap, subCateNameMap map[int64]string,
|
||||
// tools toolmod.ToolModelList,
|
||||
// articles articlemod.ArticleModelList,
|
||||
//) {
|
||||
// writeHeader(b, title, desc, keywords)
|
||||
// writeImportantPages(b, baseURL)
|
||||
// writeCategorySections(b, baseURL, categories, subCategories, true)
|
||||
//
|
||||
// if len(tools) > 0 {
|
||||
// fmt.Fprintf(b, "## AI 工具详情（最新 %d 条）\n\n", len(tools))
|
||||
// for _, t := range tools {
|
||||
// if t == nil || t.Slug == "" {
|
||||
// continue
|
||||
// }
|
||||
// fmt.Fprintf(b, "### [%s](%s)\n\n", t.Title, toolDetailURL(baseURL, t.Slug))
|
||||
//
|
||||
// summary := t.Desc
|
||||
// if summary == "" {
|
||||
// summary = t.Intro
|
||||
// }
|
||||
// if summary != "" {
|
||||
// fmt.Fprintf(b, "%s\n\n", truncate(summary, 500))
|
||||
// }
|
||||
// if subName := subCateNameMap[t.SubCateId]; subName != "" {
|
||||
// fmt.Fprintf(b, "- 子分类：[%s](%s)\n", subName, subCategoryListURL(baseURL, GetCategorySlug(t.CategoryId, categories), GetSubCategorySlug(t.SubCateId, subCategories)))
|
||||
// }
|
||||
// if t.OfficialUrl != "" {
|
||||
// fmt.Fprintf(b, "- 官网：%s\n", t.OfficialUrl)
|
||||
// }
|
||||
// if len(t.SeoInfo.Keyworld) > 0 {
|
||||
// fmt.Fprintf(b, "- 关键词：%s\n", strings.Join(t.SeoInfo.Keyworld, "、"))
|
||||
// }
|
||||
// if !t.CreatedAt.IsZero() {
|
||||
// fmt.Fprintf(b, "- 发布时间：%s\n", t.CreatedAt.Format("2006-01-02"))
|
||||
// }
|
||||
// b.WriteString("\n")
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if len(articles) > 0 {
|
||||
// fmt.Fprintf(b, "## AI 资讯详情（最新 %d 条）\n\n", len(articles))
|
||||
// for _, a := range articles {
|
||||
// if a == nil || a.Slug == "" {
|
||||
// continue
|
||||
// }
|
||||
// fmt.Fprintf(b, "### [%s](%s)\n\n", a.Title, articleDetailURL(baseURL, a.Slug))
|
||||
//
|
||||
// if a.Intro != "" {
|
||||
// fmt.Fprintf(b, "%s\n\n", truncate(a.Intro, 500))
|
||||
// }
|
||||
// if a.Author != "" {
|
||||
// fmt.Fprintf(b, "- 作者：%s\n", a.Author)
|
||||
// }
|
||||
// if len(a.SeoInfo.Keyworld) > 0 {
|
||||
// fmt.Fprintf(b, "- 关键词：%s\n", strings.Join(a.SeoInfo.Keyworld, "、"))
|
||||
// }
|
||||
// if !a.CreatedAt.IsZero() {
|
||||
// fmt.Fprintf(b, "- 发布时间：%s\n", a.CreatedAt.Format("2006-01-02"))
|
||||
// }
|
||||
// b.WriteString("\n")
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//func writeHeader(b *strings.Builder, title, desc string, keywords []string) {
|
||||
// now := time.Now().Format("2006-01-02")
|
||||
// fmt.Fprintf(b, "# %s\n\n", title)
|
||||
// if desc != "" {
|
||||
// fmt.Fprintf(b, "> %s\n\n", desc)
|
||||
// }
|
||||
// if len(keywords) > 0 {
|
||||
// fmt.Fprintf(b, "> 关键词：%s\n\n", strings.Join(keywords, "、"))
|
||||
// }
|
||||
// fmt.Fprintf(b, "> 更新时间：%s\n\n", now)
|
||||
//}
|
||||
//
|
||||
//func writeImportantPages(b *strings.Builder, baseURL string) {
|
||||
// fmt.Fprintf(b, "## 重要页面\n\n")
|
||||
// fmt.Fprintf(b, "- [首页](%s/): AI 工具导航主页\n", baseURL)
|
||||
// fmt.Fprintf(b, "- [工具列表](%s/tool/list): 全部 AI 工具\n", baseURL)
|
||||
// fmt.Fprintf(b, "- [热门推荐](%s/tool/hot): 热门 AI 工具\n", baseURL)
|
||||
// fmt.Fprintf(b, "- [AI 资讯](%s/article/list): AI 行业资讯文章\n", baseURL)
|
||||
// fmt.Fprintf(b, "- [搜索](%s/search): 搜索工具与资讯\n", baseURL)
|
||||
// fmt.Fprintf(b, "- [关于我们](%s/about/): 站点介绍\n", baseURL)
|
||||
// fmt.Fprintf(b, "- [Sitemap](%s/sitemap.xml): 网站地图\n\n", baseURL)
|
||||
//}
|
||||
//
|
||||
//func writeCategorySections(
|
||||
// b *strings.Builder,
|
||||
// baseURL string,
|
||||
// categories toolcategorymod.ToolCategoryModelList,
|
||||
// subCategories toolsubcategorymod.ToolSubCategoryModelList,
|
||||
// withSeo bool,
|
||||
//) {
|
||||
//
|
||||
// if len(subCategories) > 0 {
|
||||
// fmt.Fprintf(b, "## 工具子分类\n\n")
|
||||
// for _, s := range subCategories {
|
||||
// if s == nil {
|
||||
// continue
|
||||
// }
|
||||
// subDesc := subCategoryDesc(s, withSeo)
|
||||
// fmt.Fprintf(b, "- [%s](%s): %s\n", s.Name, subCategoryListURL(baseURL, GetCategorySlug(s.CategoryId, categories), GetSubCategorySlug(s.Id, subCategories)), subDesc)
|
||||
// }
|
||||
// b.WriteString("\n")
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//func subCategoryDesc(s *toolsubcategorymod.ToolSubCategoryModel, withSeo bool) string {
|
||||
// if withSeo && s.SeoInfo.Description != "" {
|
||||
// return s.SeoInfo.Description
|
||||
// }
|
||||
// return s.Name + "相关 AI 工具"
|
||||
//}
|
||||
|
||||
// ── URL 构建 ──────────────────────────────────────────────────
|
||||
|
||||
func toolDetailURL(baseURL, slug string) string {
|
||||
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/tool/details/%s", slug))
|
||||
}
|
||||
|
||||
func articleDetailURL(baseURL, slug string) string {
|
||||
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/article/details/%s", slug))
|
||||
}
|
||||
|
||||
func subCategoryListURL(baseURL string, categorySlug, subCateSlug string) string {
|
||||
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/classification/%s/%s", categorySlug, subCateSlug))
|
||||
}
|
||||
|
||||
// ── 工具函数 ──────────────────────────────────────────────────
|
||||
|
||||
func truncate(s string, maxLen int) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return string(runes[:maxLen]) + "…"
|
||||
}
|
||||
|
||||
func writeTextFile(filename, content string) error {
|
||||
if err := os.MkdirAll(pubdef.OutputDir, 0755); err != nil {
|
||||
return fmt.Errorf("mkdir %s: %w", pubdef.OutputDir, err)
|
||||
}
|
||||
tmpPath := filepath.Join(pubdef.OutputDir, filename+".tmp")
|
||||
f, err := os.Create(tmpPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create %s: %w", tmpPath, err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := f.WriteString(content); err != nil {
|
||||
return fmt.Errorf("write %s: %w", filename, err)
|
||||
}
|
||||
return f.Sync()
|
||||
}
|
||||
|
||||
func commitFile(filename string) error {
|
||||
src := filepath.Join(pubdef.OutputDir, filename+".tmp")
|
||||
dst := filepath.Join(pubdef.OutputDir, filename)
|
||||
if err := os.Rename(src, dst); err != nil {
|
||||
os.Remove(src)
|
||||
return fmt.Errorf("rename %s: %w", filename, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/dataReport"
|
||||
"videoweb/pub/models/eventmod"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
var isRuining = false
|
||||
|
||||
const (
|
||||
maxRetryCount = 10 // 最大重试次数
|
||||
maxReportNum = 100 // 最大上报数
|
||||
)
|
||||
|
||||
func Report() {
|
||||
//防止重入
|
||||
if isRuining {
|
||||
return
|
||||
}
|
||||
isRuining = true
|
||||
defer func() {
|
||||
isRuining = false
|
||||
}()
|
||||
ctx := context.Background()
|
||||
opts := options.Find()
|
||||
opts = opts.SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(1000)
|
||||
// 查询出一千条记录批量上传
|
||||
list, stdErr := eventmod.FindManyEvent(ctx, bson.M{
|
||||
"$or": []bson.M{
|
||||
bson.M{
|
||||
"status": eventmod.Default,
|
||||
},
|
||||
bson.M{
|
||||
"status": eventmod.Failure,
|
||||
"retryCount": bson.M{"$lte": maxRetryCount},
|
||||
},
|
||||
},
|
||||
}, opts)
|
||||
if stdErr != nil {
|
||||
log.Error("Get event list error", log.E(stdErr))
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
var eventList []*dataReport.CommonField
|
||||
for _, v := range list {
|
||||
event := &dataReport.CommonField{}
|
||||
err := json.Unmarshal([]byte(v.Data), &event)
|
||||
if err != nil {
|
||||
log.Error("event report json.Unmarshal fail", log.Any("eventData", v.Data), log.E(err))
|
||||
continue
|
||||
}
|
||||
eventList = append(eventList, event)
|
||||
if len(eventList) < maxReportNum {
|
||||
continue
|
||||
}
|
||||
// 批量上报
|
||||
err = dataReport.BatchReport(ctx, eventList)
|
||||
if err != nil {
|
||||
log.Error("Batch report error", log.E(err))
|
||||
// 将这些设置为失败
|
||||
onReportEventFail(ctx, eventList)
|
||||
} else {
|
||||
onReportEventSuccess(ctx, eventList)
|
||||
}
|
||||
eventList = []*dataReport.CommonField{} // 重新变成0个
|
||||
}
|
||||
|
||||
// 将剩余的批量上报
|
||||
err := dataReport.BatchReport(ctx, eventList)
|
||||
if err != nil {
|
||||
log.Error("Batch report error", log.E(err))
|
||||
// 将这些设置为失败
|
||||
onReportEventFail(ctx, eventList)
|
||||
} else {
|
||||
onReportEventSuccess(ctx, eventList)
|
||||
}
|
||||
}
|
||||
|
||||
func onReportEventSuccess(ctx context.Context, eventList []*dataReport.CommonField) {
|
||||
eventIds := []string{}
|
||||
for _, v := range eventList {
|
||||
eventIds = append(eventIds, v.EventID)
|
||||
}
|
||||
if len(eventIds) == 0 {
|
||||
return
|
||||
}
|
||||
err := eventmod.UpdateManyEvent(ctx, bson.M{
|
||||
"eventId": bson.M{"$in": eventIds},
|
||||
}, bson.M{
|
||||
"$set": bson.M{"status": eventmod.Success},
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("Update event fail", log.E(err))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func onReportEventFail(ctx context.Context, eventList []*dataReport.CommonField) {
|
||||
eventIds := []string{}
|
||||
for _, v := range eventList {
|
||||
eventIds = append(eventIds, v.EventID)
|
||||
}
|
||||
if len(eventIds) == 0 {
|
||||
return
|
||||
}
|
||||
err := eventmod.UpdateManyEvent(ctx, bson.M{
|
||||
"eventId": bson.M{"$in": eventIds},
|
||||
}, bson.M{
|
||||
"$set": bson.M{"status": eventmod.Failure},
|
||||
"$inc": bson.M{"retryCount": 1},
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("Update event fail", log.E(err))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
package rss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/netutil"
|
||||
"videoweb/pub/common/utils"
|
||||
"videoweb/pub/models/cfg"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/web/service/srv_cfg"
|
||||
)
|
||||
|
||||
const (
|
||||
maxRSSItems = 200 // RSS 条目上限
|
||||
)
|
||||
|
||||
var generateMu sync.Mutex
|
||||
|
||||
// RSS 2.0 结构定义
|
||||
type rssFeed struct {
|
||||
XMLName xml.Name `xml:"rss"`
|
||||
Version string `xml:"version,attr"`
|
||||
Atom string `xml:"xmlns:atom,attr"`
|
||||
Channel rssChannel `xml:"channel"`
|
||||
}
|
||||
|
||||
type rssChannel struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
Language string `xml:"language"`
|
||||
LastBuildDate string `xml:"lastBuildDate"`
|
||||
AtomLink atomLink `xml:"atom:link"`
|
||||
Items []rssItem `xml:"item"`
|
||||
}
|
||||
|
||||
type atomLink struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
Type string `xml:"type,attr"`
|
||||
}
|
||||
|
||||
type rssItem struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
GUID string `xml:"guid"`
|
||||
Enclosure *rssEnclosure `xml:"enclosure,omitempty"`
|
||||
}
|
||||
|
||||
type rssEnclosure struct {
|
||||
URL string `xml:"url,attr"`
|
||||
Type string `xml:"type,attr"`
|
||||
Length string `xml:"length,attr"`
|
||||
}
|
||||
|
||||
// rssEntry 内部排序用，合并文章与工具后统一输出
|
||||
type rssEntry struct {
|
||||
Id int64
|
||||
Title string
|
||||
Link string
|
||||
Description string
|
||||
PubAt time.Time
|
||||
Cover string
|
||||
}
|
||||
|
||||
func GenerateRSS() {
|
||||
generateMu.Lock()
|
||||
defer generateMu.Unlock()
|
||||
|
||||
ctx := context.Background()
|
||||
config, serr := srv_cfg.GetSystemConfig(ctx)
|
||||
if serr != nil {
|
||||
log.Error("GenerateRSS GetSystemConfig", log.Any("err", serr))
|
||||
return
|
||||
}
|
||||
if config == nil {
|
||||
log.Error("GenerateRSS GetSystemConfig nil config")
|
||||
return
|
||||
}
|
||||
//
|
||||
//entries := loadRSSEntries(ctx, config.WebAddr, maxRSSItems)
|
||||
//items := buildRSSItems(config, entries)
|
||||
//feedURL := utils.JoinUrlPath(config.WebAddr, "/rss.xml")
|
||||
//feed := rssFeed{
|
||||
// Version: "2.0",
|
||||
// Atom: "http://www.w3.org/2005/Atom",
|
||||
// Channel: rssChannel{
|
||||
// Title: config.WebTitle,
|
||||
// Link: config.WebAddr,
|
||||
// Description: channelDescription(config),
|
||||
// Language: "zh-cn",
|
||||
// LastBuildDate: time.Now().Format(time.RFC1123Z),
|
||||
// AtomLink: atomLink{
|
||||
// Href: feedURL,
|
||||
// Rel: "self",
|
||||
// Type: "application/rss+xml",
|
||||
// },
|
||||
// Items: items,
|
||||
// },
|
||||
//}
|
||||
|
||||
//const filename = "rss.xml"
|
||||
//if err := writeRSSFile(filename, feed); err != nil {
|
||||
// log.Error("GenerateRSS write", log.Any("err", err))
|
||||
// return
|
||||
//}
|
||||
//
|
||||
//src := filepath.Join(pubdef.OutputDir, filename+".tmp")
|
||||
//dst := filepath.Join(pubdef.OutputDir, filename)
|
||||
//if err := os.Rename(src, dst); err != nil {
|
||||
// log.Error("GenerateRSS rename", log.Any("err", err))
|
||||
// os.Remove(src)
|
||||
// return
|
||||
//}
|
||||
|
||||
//log.Info("GenerateRSS done", log.Any("items", len(items)))
|
||||
}
|
||||
|
||||
func channelDescription(config *cfg.SysCfg) string {
|
||||
if config.WebDescription != "" {
|
||||
return config.WebDescription
|
||||
}
|
||||
return "AI 工具导航与资讯更新"
|
||||
}
|
||||
|
||||
//
|
||||
//func loadRSSEntries(ctx context.Context, baseURL string, limit int) []rssEntry {
|
||||
// baseURL = strings.TrimSuffix(baseURL, "/")
|
||||
// articleFilter := bson.M{
|
||||
// articlemod.FN.Article.ReviewStatus: 1,
|
||||
// articlemod.FN.Article.Slug: bson.M{"$nin": []interface{}{"", nil}},
|
||||
// }
|
||||
// articleOpt := options.Find().
|
||||
// SetSort(bson.D{{Key: articlemod.FN.Article.CreatedAt, Value: -1}}).
|
||||
// SetLimit(int64(limit))
|
||||
// articles, _ := articlemod.FindMany(ctx, articleFilter, articleOpt)
|
||||
//
|
||||
// toolFilter := bson.M{
|
||||
// toolmod.FN.Tool.ReviewStatus: 1,
|
||||
// toolmod.FN.Tool.Slug: bson.M{"$nin": []interface{}{"", nil}},
|
||||
// }
|
||||
// toolOpt := options.Find().
|
||||
// SetSort(bson.D{{Key: toolmod.FN.Tool.CreatedAt, Value: -1}}).
|
||||
// SetLimit(int64(limit))
|
||||
// tools, _ := toolmod.FindMany(ctx, toolFilter, toolOpt)
|
||||
//
|
||||
// entries := make([]rssEntry, 0, len(articles)+len(tools))
|
||||
// for _, a := range articles {
|
||||
// if a == nil || a.Slug == "" {
|
||||
// continue
|
||||
// }
|
||||
// desc := a.Intro
|
||||
// if desc == "" {
|
||||
// desc = a.Title
|
||||
// }
|
||||
// entries = append(entries, rssEntry{
|
||||
// Id: a.Id,
|
||||
// Title: a.Title,
|
||||
// Link: articleDetailURL(baseURL, a.Slug),
|
||||
// Description: desc,
|
||||
// PubAt: a.CreatedAt,
|
||||
// Cover: a.Cover,
|
||||
// })
|
||||
// }
|
||||
// for _, t := range tools {
|
||||
// if t == nil || t.Slug == "" {
|
||||
// continue
|
||||
// }
|
||||
// desc := t.Intro
|
||||
// if desc == "" {
|
||||
// desc = t.Desc
|
||||
// }
|
||||
// if desc == "" {
|
||||
// desc = t.Title
|
||||
// }
|
||||
// cover := t.Logo
|
||||
// if cover == "" {
|
||||
// cover = t.Snapshot
|
||||
// }
|
||||
// entries = append(entries, rssEntry{
|
||||
// Id: t.Id,
|
||||
// Title: t.Title,
|
||||
// Link: toolDetailURL(baseURL, t.Slug),
|
||||
// Description: desc,
|
||||
// PubAt: t.CreatedAt,
|
||||
// Cover: cover,
|
||||
// })
|
||||
// }
|
||||
//
|
||||
// sort.Slice(entries, func(i, j int) bool {
|
||||
// return entries[i].PubAt.After(entries[j].PubAt)
|
||||
// })
|
||||
// if len(entries) > limit {
|
||||
// entries = entries[:limit]
|
||||
// }
|
||||
// return entries
|
||||
//}
|
||||
//
|
||||
//func buildRSSItems(config *cfg.SysCfg, entries []rssEntry) []rssItem {
|
||||
// logoURL := utils.JoinUrlPath(strings.TrimSuffix(config.WebAddr, "/"), "/static/images/logo.png")
|
||||
// imgCdn := imageCdn(config)
|
||||
//
|
||||
// items := make([]rssItem, 0, len(entries))
|
||||
// for _, e := range entries {
|
||||
// if e.Link == "" {
|
||||
// continue
|
||||
// }
|
||||
//
|
||||
// pubDate := ""
|
||||
// if !e.PubAt.IsZero() {
|
||||
// pubDate = e.PubAt.Format(time.RFC1123Z)
|
||||
// }
|
||||
//
|
||||
// item := rssItem{
|
||||
// Title: e.Title,
|
||||
// Link: e.Link,
|
||||
// Description: e.Description,
|
||||
// PubDate: pubDate,
|
||||
// GUID: e.Link,
|
||||
// }
|
||||
//
|
||||
// encURL, encType := resolveEnclosureURL(config, imgCdn, logoURL, e.Cover, e.Id)
|
||||
// item.Enclosure = &rssEnclosure{
|
||||
// URL: encURL,
|
||||
// Type: encType,
|
||||
// Length: "0",
|
||||
// }
|
||||
// items = append(items, item)
|
||||
// }
|
||||
// return items
|
||||
//}
|
||||
|
||||
func imageCdn(config *cfg.SysCfg) string {
|
||||
if config.ImgWebCdn != "" {
|
||||
return config.ImgWebCdn
|
||||
}
|
||||
return config.UnencryptedCdn
|
||||
}
|
||||
|
||||
func resolveEnclosureURL(config *cfg.SysCfg, imgCdn, logoURL, cover string, id int64) (url, mime string) {
|
||||
mime = "image/jpeg"
|
||||
if cover != "" && imgCdn != "" {
|
||||
url = netutil.MergeURL(imgCdn, cover)
|
||||
} else if len(config.SitemapDefaultImages) > 0 && imgCdn != "" {
|
||||
img := config.SitemapDefaultImages[absModIndex(id, len(config.SitemapDefaultImages))]
|
||||
url = netutil.MergeURL(imgCdn, img)
|
||||
} else {
|
||||
url = logoURL
|
||||
mime = "image/png"
|
||||
}
|
||||
if strings.HasSuffix(strings.ToLower(url), ".png") {
|
||||
mime = "image/png"
|
||||
} else if strings.HasSuffix(strings.ToLower(url), ".webp") {
|
||||
mime = "image/webp"
|
||||
}
|
||||
return url, mime
|
||||
}
|
||||
|
||||
func toolDetailURL(baseURL, slug string) string {
|
||||
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/tool/details/%s", slug))
|
||||
}
|
||||
|
||||
func articleDetailURL(baseURL, slug string) string {
|
||||
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/article/details/%s", slug))
|
||||
}
|
||||
|
||||
func absModIndex(id int64, length int) int {
|
||||
if length <= 0 {
|
||||
return 0
|
||||
}
|
||||
idx := id % int64(length)
|
||||
if idx < 0 {
|
||||
idx = -idx
|
||||
}
|
||||
return int(idx)
|
||||
}
|
||||
|
||||
func writeRSSFile(filename string, data interface{}) error {
|
||||
if err := os.MkdirAll(pubdef.OutputDir, 0755); err != nil {
|
||||
return fmt.Errorf("mkdir %s: %w", pubdef.OutputDir, err)
|
||||
}
|
||||
path := filepath.Join(pubdef.OutputDir, filename+".tmp")
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create %s: %w", filename, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if _, err := file.WriteString(xml.Header); err != nil {
|
||||
return fmt.Errorf("write header %s: %w", filename, err)
|
||||
}
|
||||
encoder := xml.NewEncoder(file)
|
||||
encoder.Indent("", " ")
|
||||
if err := encoder.Encode(data); err != nil {
|
||||
return fmt.Errorf("encode %s: %w", filename, err)
|
||||
}
|
||||
if err := encoder.Flush(); err != nil {
|
||||
return fmt.Errorf("flush %s: %w", filename, err)
|
||||
}
|
||||
return file.Sync()
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
// Package seopush 为搜索引擎推送做兜底：补推已上架短剧，并重试失败的任务。
|
||||
package seopush
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/models/playletmod"
|
||||
"videoweb/pub/models/seopushmod"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/web/service/srv_seopush"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const (
|
||||
backfillLimit = 100
|
||||
retryLimit = 50
|
||||
taskKeepDays = 30
|
||||
)
|
||||
|
||||
var jobMu sync.Mutex
|
||||
|
||||
// Run 定时兜底任务
|
||||
func Run() {
|
||||
if !jobMu.TryLock() {
|
||||
log.Warn("seopush: skip job, previous run still in progress")
|
||||
return
|
||||
}
|
||||
defer jobMu.Unlock()
|
||||
|
||||
ctx := context.Background()
|
||||
if !pubdef.Enabled() {
|
||||
discardPendingTasks(ctx)
|
||||
return
|
||||
}
|
||||
webAddr := srv_seopush.SEOBaseURL(ctx)
|
||||
if webAddr == "" {
|
||||
return
|
||||
}
|
||||
backfillVisiblePlaylets(ctx)
|
||||
retryPendingTasks(ctx, webAddr)
|
||||
cleanupExpiredTasks(ctx)
|
||||
}
|
||||
|
||||
func discardPendingTasks(ctx context.Context) {
|
||||
filter := bson.M{"$or": []bson.M{
|
||||
{seopushmod.FN.SeoPushTask.GoogleStatus: seopushmod.PushStatusPending},
|
||||
{seopushmod.FN.SeoPushTask.IndexNowStatus: seopushmod.PushStatusPending},
|
||||
}}
|
||||
list, serr := seopushmod.FindManySeoPushTasks(ctx, filter)
|
||||
if serr != nil {
|
||||
log.Warn("seopush: find pending tasks to discard failed", log.E(serr))
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
ids := make([]int64, 0, len(list))
|
||||
seen := make(map[int64]struct{}, len(list))
|
||||
for _, task := range list {
|
||||
if task == nil || task.VideoId <= 0 {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[task.VideoId]; ok {
|
||||
continue
|
||||
}
|
||||
seen[task.VideoId] = struct{}{}
|
||||
ids = append(ids, task.VideoId)
|
||||
}
|
||||
count, serr := seopushmod.DelManySeoPushTask(ctx, filter)
|
||||
if serr != nil {
|
||||
log.Warn("seopush: discard pending tasks failed", log.E(serr))
|
||||
return
|
||||
}
|
||||
if len(ids) > 0 {
|
||||
if serr := playletmod.UpdateManyPlaylet(ctx,
|
||||
bson.M{playletmod.FN.Playlet.Id: bson.M{"$in": ids}},
|
||||
bson.M{"$set": bson.M{playletmod.FN.Playlet.IndexNow: int64(0)}}); serr != nil {
|
||||
log.Warn("seopush: reset playlet indexNow failed", log.E(serr))
|
||||
}
|
||||
}
|
||||
log.Info("seopush=skip=disabled", log.Any("discarded", count))
|
||||
}
|
||||
|
||||
func backfillVisiblePlaylets(ctx context.Context) {
|
||||
filter := bson.M{
|
||||
playletmod.FN.Playlet.Status: playletmod.StatusUp,
|
||||
"$or": []bson.M{
|
||||
{playletmod.FN.Playlet.IndexNow: 0},
|
||||
{playletmod.FN.Playlet.IndexNow: bson.M{"$exists": false}},
|
||||
},
|
||||
}
|
||||
opt := options.Find().
|
||||
SetSort(bson.M{playletmod.FN.Playlet.OnlineAt: -1}).
|
||||
SetLimit(backfillLimit)
|
||||
|
||||
list, serr := playletmod.FindManyPlaylet(ctx, filter, opt)
|
||||
if serr != nil {
|
||||
log.Warn("seopush: backfill find playlets failed", log.E(serr))
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
for _, item := range list {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
srv_seopush.RegisterPlayletUpdated(ctx, item.Id, item.Status, item.OnlineAt)
|
||||
}
|
||||
log.Info("seopush: backfill registered", log.Any("count", len(list)))
|
||||
}
|
||||
|
||||
func retryPendingTasks(ctx context.Context, webAddr string) {
|
||||
conds := []bson.M{
|
||||
{
|
||||
seopushmod.FN.SeoPushTask.GoogleStatus: seopushmod.PushStatusPending,
|
||||
seopushmod.FN.SeoPushTask.GoogleRetry: bson.M{"$lt": seopushmod.MaxPushRetry},
|
||||
},
|
||||
{
|
||||
seopushmod.FN.SeoPushTask.IndexNowStatus: seopushmod.PushStatusPending,
|
||||
seopushmod.FN.SeoPushTask.IndexNowRetry: bson.M{"$lt": seopushmod.MaxPushRetry},
|
||||
},
|
||||
}
|
||||
opt := options.Find().
|
||||
SetSort(bson.M{seopushmod.FN.SeoPushTask.CreatedAt: 1}).
|
||||
SetLimit(retryLimit)
|
||||
taskList, serr := seopushmod.FindManySeoPushTasks(ctx, bson.M{"$or": conds}, opt)
|
||||
if serr != nil {
|
||||
log.Warn("seopush: find pending tasks failed", log.E(serr))
|
||||
return
|
||||
}
|
||||
if len(taskList) == 0 {
|
||||
return
|
||||
}
|
||||
for _, task := range taskList {
|
||||
if task == nil || strings.Contains(task.Url, "?") {
|
||||
continue
|
||||
}
|
||||
srv_seopush.PushTask(ctx, webAddr, task)
|
||||
}
|
||||
log.Info("seopush: pending tasks processed", log.Any("count", len(taskList)))
|
||||
}
|
||||
|
||||
func cleanupExpiredTasks(ctx context.Context) {
|
||||
deadline := time.Now().AddDate(0, 0, -taskKeepDays)
|
||||
count, serr := seopushmod.DelManySeoPushTask(ctx, bson.M{
|
||||
seopushmod.FN.SeoPushTask.UpdatedAt: bson.M{"$lt": deadline},
|
||||
})
|
||||
if serr != nil {
|
||||
log.Warn("seopush: cleanup expired tasks failed", log.E(serr))
|
||||
return
|
||||
}
|
||||
if count > 0 {
|
||||
log.Info("seopush: expired tasks cleaned", log.Any("count", count))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package job
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"videoweb/pub/common/laosiji"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/models/videoinformationmod"
|
||||
"videoweb/web/service/srv_videoinfo"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const transcodeBatchSize = 5
|
||||
const transcodeMaxFailCount = 3
|
||||
|
||||
var transcodeRunning int32
|
||||
var transcodeCheckRunning int32
|
||||
|
||||
const transcodeCheckBatchSize = 100
|
||||
|
||||
// TranscodeVideoByWatchTimes 按观看次数挑选尚未转码成功的视频，批量提交 H265 转码。
|
||||
func TranscodeVideoByWatchTimes() {
|
||||
if !atomic.CompareAndSwapInt32(&transcodeRunning, 0, 1) {
|
||||
log.Info("TranscodeVideoByWatchTimes skip: previous run still running")
|
||||
return
|
||||
}
|
||||
defer atomic.StoreInt32(&transcodeRunning, 0)
|
||||
|
||||
ctx := context.Background()
|
||||
if laosiji.LaosijiClient == nil || laosiji.LaosijiDomains == nil {
|
||||
log.Warn("TranscodeVideoByWatchTimes skip: laosiji not ready")
|
||||
return
|
||||
}
|
||||
if _, err := laosiji.LaosijiDomains.Get(ctx); err != nil {
|
||||
log.Warn("TranscodeVideoByWatchTimes skip: domains not ready", log.E(err))
|
||||
return
|
||||
}
|
||||
|
||||
filter := bson.M{
|
||||
videoinformationmod.FN.VideoInformation.HevcStatus: bson.M{
|
||||
"$in": bson.A{videoinformationmod.HevcStatusNone, videoinformationmod.HevcStatusFailed},
|
||||
},
|
||||
videoinformationmod.FN.VideoInformation.HevcFailCount: bson.M{"$lt": transcodeMaxFailCount},
|
||||
videoinformationmod.FN.VideoInformation.Video.Url: bson.M{"$ne": ""},
|
||||
}
|
||||
opts := options.Find().
|
||||
SetSort(bson.D{{Key: videoinformationmod.FN.VideoInformation.Real.WatchTimes, Value: -1}}).
|
||||
SetLimit(transcodeBatchSize)
|
||||
|
||||
videoList, serr := videoinformationmod.FindManyVideoInformations(ctx, filter, opts)
|
||||
if serr != nil {
|
||||
log.Error("TranscodeVideoByWatchTimes find video failed", log.Any("serr", serr))
|
||||
return
|
||||
}
|
||||
if len(videoList) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("TranscodeVideoByWatchTimes start", log.Any("count", len(videoList)))
|
||||
for _, v := range videoList {
|
||||
src := strings.TrimSpace(v.Video.Url)
|
||||
if src == "" {
|
||||
continue
|
||||
}
|
||||
if err := srv_videoinfo.SubmitTranscodeVideo(ctx, v.Id, src, v.Video.FileId); err != nil {
|
||||
log.Warn("TranscodeVideoByWatchTimes submit failed",
|
||||
log.Any("videoId", v.Id), log.Any("watchTimes", v.Real.WatchTimes), log.E(err))
|
||||
}
|
||||
}
|
||||
log.Info("TranscodeVideoByWatchTimes done", log.Any("count", len(videoList)))
|
||||
}
|
||||
|
||||
// CheckTranscodingResults 查询转码中任务并回写结果。
|
||||
func CheckTranscodingResults() {
|
||||
if !atomic.CompareAndSwapInt32(&transcodeCheckRunning, 0, 1) {
|
||||
log.Info("CheckTranscodingResults skip: previous run still running")
|
||||
return
|
||||
}
|
||||
defer atomic.StoreInt32(&transcodeCheckRunning, 0)
|
||||
|
||||
ctx := context.Background()
|
||||
if laosiji.LaosijiClient == nil || laosiji.LaosijiDomains == nil {
|
||||
return
|
||||
}
|
||||
|
||||
filter := bson.M{videoinformationmod.FN.VideoInformation.HevcStatus: videoinformationmod.HevcStatusPending}
|
||||
opts := options.Find().SetLimit(transcodeCheckBatchSize)
|
||||
videoList, serr := videoinformationmod.FindManyVideoInformations(ctx, filter, opts)
|
||||
if serr != nil {
|
||||
log.Error("CheckTranscodingResults find video failed", log.Any("serr", serr))
|
||||
return
|
||||
}
|
||||
if len(videoList) == 0 {
|
||||
return
|
||||
}
|
||||
for _, v := range videoList {
|
||||
src := strings.TrimSpace(v.Video.Url)
|
||||
if src == "" {
|
||||
continue
|
||||
}
|
||||
if err := srv_videoinfo.RefreshTranscodeVideo(ctx, v.Id, src, v.Video.FileId); err != nil {
|
||||
log.Warn("CheckTranscodingResults refresh failed", log.Any("videoId", v.Id), log.E(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package vipexpiry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/models/usermod"
|
||||
"videoweb/pub/pubdef"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
var isRunning = false
|
||||
|
||||
// ResetExpiredVip 将 CurrentVipExpire 已过期但 VipType 仍非 None 的用户批量重置为普通用户。
|
||||
// 同时清空 ShowCard* 展示字段，保证 DB 数据与实际状态一致。
|
||||
func ResetExpiredVip() {
|
||||
if isRunning {
|
||||
return
|
||||
}
|
||||
isRunning = true
|
||||
defer func() { isRunning = false }()
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
|
||||
filter := bson.M{
|
||||
usermod.FN.User.VipType: bson.M{"$gt": pubdef.VipTypeNone},
|
||||
usermod.FN.User.CurrentVipExpire: bson.M{"$lt": now},
|
||||
}
|
||||
update := bson.M{
|
||||
"$set": bson.M{
|
||||
usermod.FN.User.VipType: pubdef.VipTypeNone,
|
||||
usermod.FN.User.ShowCardId: int64(0),
|
||||
usermod.FN.User.ShowCardName: "",
|
||||
usermod.FN.User.UpdatedAt: now,
|
||||
},
|
||||
}
|
||||
|
||||
if serr := usermod.UpdateManyUser(ctx, filter, update); serr != nil {
|
||||
log.Error("vipexpiry:ResetExpiredVip UpdateManyUser failed", log.Any("err", serr))
|
||||
return
|
||||
}
|
||||
log.Info("vipexpiry:ResetExpiredVip done")
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package playletwatermark
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/srvdef"
|
||||
"videoweb/pub/models/playletchaptermod"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
const watermarkStuckTimeout = 48 * time.Hour
|
||||
|
||||
// MarkStuckChaptersFailed 兜底：已登记/处理中超过 2 天仍未完成的剧集标记为处理失败。
|
||||
// media 原片已 CompletedFail 时由 submit 立即回写，不必等这 2 天。
|
||||
func MarkStuckChaptersFailed() {
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
cutoff := now.Add(-watermarkStuckTimeout)
|
||||
err := playletchaptermod.UpdateManyPlayletChapter(ctx, bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: bson.M{
|
||||
"$in": []playletchaptermod.WatermarkStatus{
|
||||
playletchaptermod.WatermarkStatusRegistered,
|
||||
playletchaptermod.WatermarkStatusProcessing,
|
||||
},
|
||||
},
|
||||
playletchaptermod.FN.PlayletChapter.UpdatedAt: bson.M{"$lt": cutoff},
|
||||
}, bson.M{"$set": bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: playletchaptermod.WatermarkStatusFailed,
|
||||
playletchaptermod.FN.PlayletChapter.UpdatedAt: now,
|
||||
}})
|
||||
if err != nil && err.Code != srvdef.ErrNoData {
|
||||
log.Error("playletwatermark:MarkStuckChaptersFailed 更新失败", log.E(err))
|
||||
return
|
||||
}
|
||||
log.Info("playletwatermark:MarkStuckChaptersFailed end", log.Any("cutoff", cutoff))
|
||||
}
|
||||
@@ -0,0 +1,471 @@
|
||||
package playletwatermark
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/models/cfg"
|
||||
"videoweb/pub/models/playletchaptermod"
|
||||
"videoweb/pub/models/playletmod"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/pub/storage/proxy"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const (
|
||||
submitBatchSize = 10
|
||||
taskAddRetryBatch = 20
|
||||
taskAddScanLimit = 200
|
||||
mediaTaskAlreadyExists = 6007
|
||||
mediaTaskSourceNotReady = 6015
|
||||
sourceNotReadyRetryWait = 5 * time.Minute
|
||||
mediaFileVersionV1 = "v1"
|
||||
mediaFileVersionV2 = "v2"
|
||||
)
|
||||
|
||||
// watermarkSubmitResult 是一次 task/add 或重新加水印的结果。
|
||||
type watermarkSubmitResult int
|
||||
|
||||
const (
|
||||
watermarkSubmitFailed watermarkSubmitResult = iota
|
||||
watermarkSubmitNotReady
|
||||
watermarkSubmitOK
|
||||
)
|
||||
|
||||
var (
|
||||
submitRunning bool
|
||||
submitLock sync.Mutex
|
||||
)
|
||||
|
||||
// SubmitPendingChapters 定时提交未加水印剧集到 media（每10分钟最多10条）
|
||||
func SubmitPendingChapters() {
|
||||
submitLock.Lock()
|
||||
if submitRunning {
|
||||
submitLock.Unlock()
|
||||
return
|
||||
}
|
||||
submitRunning = true
|
||||
submitLock.Unlock()
|
||||
defer func() {
|
||||
submitLock.Lock()
|
||||
submitRunning = false
|
||||
submitLock.Unlock()
|
||||
}()
|
||||
|
||||
ctx := context.Background()
|
||||
log.Info("playletwatermark:SubmitPendingChapters begin")
|
||||
//MarkStuckChaptersFailed()
|
||||
|
||||
sysCfg, err := cfg.FindOneSysCfg(ctx, bson.M{cfg.FN.SysCfg.Id: 1})
|
||||
if err != nil || sysCfg == nil {
|
||||
log.Error("playletwatermark:SubmitPendingChapters 读取系统配置失败", log.E(err))
|
||||
return
|
||||
}
|
||||
if !hasVideoWatermark(sysCfg) {
|
||||
log.Warn("playletwatermark:SubmitPendingChapters 水印素材未配置，跳过")
|
||||
return
|
||||
}
|
||||
cfg.CacheAppKey(sysCfg.AppKey)
|
||||
|
||||
registerPendingSources(ctx)
|
||||
submitWatermarkTasks(ctx, sysCfg)
|
||||
log.Info("playletwatermark:SubmitPendingChapters end")
|
||||
}
|
||||
|
||||
// registerPendingSources 将未处理剧集的相对 m3u8 登记到 media
|
||||
func registerPendingSources(ctx context.Context) {
|
||||
list, err := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: bson.M{
|
||||
"$in": []playletchaptermod.WatermarkStatus{
|
||||
playletchaptermod.WatermarkStatusPending,
|
||||
playletchaptermod.WatermarkStatusRegistered,
|
||||
},
|
||||
},
|
||||
playletchaptermod.FN.PlayletChapter.VideoUrl: bson.M{"$gt": ""},
|
||||
"$or": []bson.M{
|
||||
{playletchaptermod.FN.PlayletChapter.MediaSourceId: bson.M{"$exists": false}},
|
||||
{playletchaptermod.FN.PlayletChapter.MediaSourceId: ""},
|
||||
},
|
||||
}, options.Find().
|
||||
SetSort(bson.D{{Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1}}).
|
||||
SetLimit(submitBatchSize))
|
||||
if err != nil {
|
||||
log.Error("playletwatermark:registerPendingSources 查询失败", log.E(err))
|
||||
return
|
||||
}
|
||||
for _, item := range list {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
sourceID, syncErr := RegisterChapterSource(item)
|
||||
if syncErr != nil {
|
||||
log.Error("playletwatermark:registerPendingSources SyncMediaFile 失败",
|
||||
log.Any("chapterId", item.Id), log.Any("videoUrl", item.VideoUrl), log.E(syncErr))
|
||||
continue
|
||||
}
|
||||
now := time.Now()
|
||||
if updateErr := playletchaptermod.UpdateOnePlayletChapter(ctx,
|
||||
bson.M{playletchaptermod.FN.PlayletChapter.Id: item.Id},
|
||||
bson.M{"$set": bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.MediaSourceId: sourceID,
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: playletchaptermod.WatermarkStatusRegistered,
|
||||
playletchaptermod.FN.PlayletChapter.UpdatedAt: now,
|
||||
}}); updateErr != nil {
|
||||
log.Error("playletwatermark:registerPendingSources 更新章节失败",
|
||||
log.Any("chapterId", item.Id), log.E(updateErr))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterChapterSource 将剧集原片登记到 media 切片服，返回 mediaSourceId。
|
||||
func RegisterChapterSource(item *playletchaptermod.PlayletChapter) (string, error) {
|
||||
if item == nil {
|
||||
return "", errors.New("chapter is nil")
|
||||
}
|
||||
fieldPath, version := normalizeMediaFilePath(item.VideoUrl)
|
||||
if fieldPath == "" {
|
||||
return "", errors.New("invalid video path")
|
||||
}
|
||||
sourceID := primitive.NewObjectID()
|
||||
req := &proxy.MediaFileSyncReq{
|
||||
AppId: pubdef.AppId,
|
||||
AppKey: cfg.CachedAppKey(context.Background()),
|
||||
VideoList: []proxy.MediaFileSyncItem{
|
||||
{
|
||||
ID: sourceID.Hex(),
|
||||
Title: mediaWatermarkTitle(playletForChapter(item), item),
|
||||
TaskID: sourceID.Hex(),
|
||||
FieldNameFs: fieldPath,
|
||||
Width: item.Width,
|
||||
Height: item.Height,
|
||||
VideoTime: item.MediaTime,
|
||||
Size: item.MediaSize,
|
||||
MediaFileVersion: version,
|
||||
IsPlaylet: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
result, syncErr := proxy.SyncMediaFile(req, "")
|
||||
if syncErr != nil {
|
||||
return "", syncErr
|
||||
}
|
||||
if result == nil || result.Code != proxy.Success {
|
||||
return "", errors.New("SyncMediaFile failed")
|
||||
}
|
||||
return sourceID.Hex(), nil
|
||||
}
|
||||
|
||||
// submitWatermarkTasks 对已到重试时间的已登记原片尝试 task/add。
|
||||
// 同一轮按重试时间、剧集 id 最多检查 taskAddScanLimit 条，成功提交满 taskAddRetryBatch 条即停。
|
||||
// 原片未处理完的剧集延后，不再占住队头。
|
||||
func submitWatermarkTasks(ctx context.Context, sysCfg *cfg.SysCfg) {
|
||||
now := time.Now()
|
||||
list, err := playletchaptermod.FindManyPlayletChapter(ctx, dueRegisteredWatermarkFilter(now), options.Find().
|
||||
SetSort(bson.D{
|
||||
{Key: playletchaptermod.FN.PlayletChapter.WatermarkRetryAt, Value: 1},
|
||||
{Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1},
|
||||
}).
|
||||
SetLimit(taskAddScanLimit))
|
||||
if err != nil {
|
||||
log.Error("playletwatermark:submitWatermarkTasks 查询失败", log.E(err))
|
||||
return
|
||||
}
|
||||
submitted := 0
|
||||
scanned := 0
|
||||
for _, item := range list {
|
||||
if submitted >= taskAddRetryBatch {
|
||||
break
|
||||
}
|
||||
if item == nil || item.MediaSourceId == "" {
|
||||
continue
|
||||
}
|
||||
scanned++
|
||||
var result watermarkSubmitResult
|
||||
if strings.TrimSpace(item.MediaTaskId) != "" {
|
||||
result = restartWatermarkTask(ctx, item, sysCfg)
|
||||
} else {
|
||||
result = addWatermarkTask(ctx, item, sysCfg)
|
||||
}
|
||||
switch result {
|
||||
case watermarkSubmitNotReady:
|
||||
postponeWatermarkRetry(ctx, item, now)
|
||||
case watermarkSubmitOK:
|
||||
if markWatermarkProcessing(ctx, item, now) {
|
||||
submitted++
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Info("playletwatermark:submitWatermarkTasks end",
|
||||
log.Any("scanned", scanned),
|
||||
log.Any("submitted", submitted),
|
||||
log.Any("candidates", len(list)))
|
||||
}
|
||||
|
||||
// dueRegisteredWatermarkFilter 选出已登记、且到了重试时间的剧集。
|
||||
// 没有重试时间的记录优先，原片未就绪被延后的记录排在后面。
|
||||
func dueRegisteredWatermarkFilter(now time.Time) bson.M {
|
||||
return bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: playletchaptermod.WatermarkStatusRegistered,
|
||||
playletchaptermod.FN.PlayletChapter.MediaSourceId: bson.M{"$gt": ""},
|
||||
"$or": []bson.M{
|
||||
{playletchaptermod.FN.PlayletChapter.WatermarkRetryAt: bson.M{"$exists": false}},
|
||||
{playletchaptermod.FN.PlayletChapter.WatermarkRetryAt: bson.M{"$lte": now}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func classifyMediaTaskCode(code int) watermarkSubmitResult {
|
||||
switch code {
|
||||
case proxy.Success, mediaTaskAlreadyExists:
|
||||
return watermarkSubmitOK
|
||||
case mediaTaskSourceNotReady:
|
||||
return watermarkSubmitNotReady
|
||||
default:
|
||||
return watermarkSubmitFailed
|
||||
}
|
||||
}
|
||||
|
||||
func markWatermarkProcessing(ctx context.Context, item *playletchaptermod.PlayletChapter, now time.Time) bool {
|
||||
if updateErr := playletchaptermod.UpdateOnePlayletChapter(ctx,
|
||||
bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.Id: item.Id,
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: playletchaptermod.WatermarkStatusRegistered,
|
||||
},
|
||||
bson.M{
|
||||
"$set": bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: playletchaptermod.WatermarkStatusProcessing,
|
||||
playletchaptermod.FN.PlayletChapter.UpdatedAt: now,
|
||||
},
|
||||
"$unset": bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkRetryAt: "",
|
||||
},
|
||||
}); updateErr != nil {
|
||||
log.Error("playletwatermark:submitWatermarkTasks 更新为处理中失败",
|
||||
log.Any("chapterId", item.Id),
|
||||
log.E(updateErr))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func postponeWatermarkRetry(ctx context.Context, item *playletchaptermod.PlayletChapter, now time.Time) {
|
||||
retryAt := now.Add(sourceNotReadyRetryWait)
|
||||
if updateErr := playletchaptermod.UpdateOnePlayletChapter(ctx,
|
||||
bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.Id: item.Id,
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: playletchaptermod.WatermarkStatusRegistered,
|
||||
},
|
||||
bson.M{"$set": bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkRetryAt: retryAt,
|
||||
}}); updateErr != nil {
|
||||
log.Error("playletwatermark:submitWatermarkTasks 延后重试失败",
|
||||
log.Any("chapterId", item.Id),
|
||||
log.E(updateErr))
|
||||
return
|
||||
}
|
||||
log.Info("playletwatermark:submitWatermarkTasks 原片未处理完，延后重试",
|
||||
log.Any("chapterId", item.Id),
|
||||
log.Any("mediaSourceId", item.MediaSourceId),
|
||||
log.Any("retryAt", retryAt))
|
||||
}
|
||||
|
||||
func addWatermarkTask(ctx context.Context, item *playletchaptermod.PlayletChapter, sysCfg *cfg.SysCfg) watermarkSubmitResult {
|
||||
req := &proxy.MediaTaskAddReq{
|
||||
MediaSourceId: item.MediaSourceId,
|
||||
AppId: pubdef.AppId,
|
||||
AppKey: strings.TrimSpace(sysCfg.AppKey),
|
||||
Title: mediaWatermarkTitle(playletForChapter(item), item),
|
||||
WatermarkImage: sysCfg.WatermarkImage,
|
||||
WatermarkMedia: sysCfg.WatermarkMedia,
|
||||
WatermarkMediaTop: sysCfg.WatermarkMediaTop,
|
||||
WatermarkMediaTopRight: sysCfg.WatermarkMediaTopRight,
|
||||
WatermarkMediaBottomLeft: sysCfg.WatermarkMediaBottomLeft,
|
||||
}
|
||||
result, addErr := proxy.AddMediaTask(req, "")
|
||||
if addErr != nil || result == nil {
|
||||
log.Info("playletwatermark:submitWatermarkTasks task/add 请求失败",
|
||||
log.Any("chapterId", item.Id),
|
||||
log.Any("mediaSourceId", item.MediaSourceId),
|
||||
log.E(addErr))
|
||||
return watermarkSubmitFailed
|
||||
}
|
||||
// 200=新建成功；6007=任务已存在。6015=原片未处理完，延后并不占住本轮名额。
|
||||
switch classifyMediaTaskCode(result.Code) {
|
||||
case watermarkSubmitOK:
|
||||
return watermarkSubmitOK
|
||||
case watermarkSubmitNotReady:
|
||||
return watermarkSubmitNotReady
|
||||
default:
|
||||
log.Info("playletwatermark:submitWatermarkTasks task/add 暂未成功",
|
||||
log.Any("chapterId", item.Id),
|
||||
log.Any("mediaSourceId", item.MediaSourceId),
|
||||
log.Any("result", result))
|
||||
markFailedIfMediaSourceFailed(ctx, item)
|
||||
return watermarkSubmitFailed
|
||||
}
|
||||
}
|
||||
|
||||
func restartWatermarkTask(ctx context.Context, item *playletchaptermod.PlayletChapter, sysCfg *cfg.SysCfg) watermarkSubmitResult {
|
||||
watermarkImage := sysCfg.WatermarkImage
|
||||
watermarkMedia := sysCfg.WatermarkMedia
|
||||
watermarkMediaTop := sysCfg.WatermarkMediaTop
|
||||
watermarkMediaTopRight := sysCfg.WatermarkMediaTopRight
|
||||
watermarkMediaBottomLeft := sysCfg.WatermarkMediaBottomLeft
|
||||
req := &proxy.MediaTaskUpdateWatermarkReq{
|
||||
Id: item.MediaTaskId,
|
||||
AppId: pubdef.AppId,
|
||||
AppKey: strings.TrimSpace(sysCfg.AppKey),
|
||||
WatermarkImage: &watermarkImage,
|
||||
WatermarkMedia: &watermarkMedia,
|
||||
WatermarkMediaTop: &watermarkMediaTop,
|
||||
WatermarkMediaTopRight: &watermarkMediaTopRight,
|
||||
WatermarkMediaBottomLeft: &watermarkMediaBottomLeft,
|
||||
}
|
||||
result, updateErr := proxy.UpdateMediaWatermark(req, "")
|
||||
if updateErr != nil || result == nil {
|
||||
log.Info("playletwatermark:submitWatermarkTasks update/watermark 请求失败",
|
||||
log.Any("chapterId", item.Id),
|
||||
log.Any("mediaTaskId", item.MediaTaskId),
|
||||
log.E(updateErr))
|
||||
return watermarkSubmitFailed
|
||||
}
|
||||
switch classifyMediaTaskCode(result.Code) {
|
||||
case watermarkSubmitOK:
|
||||
return watermarkSubmitOK
|
||||
case watermarkSubmitNotReady:
|
||||
return watermarkSubmitNotReady
|
||||
default:
|
||||
log.Info("playletwatermark:submitWatermarkTasks update/watermark 暂未成功",
|
||||
log.Any("chapterId", item.Id),
|
||||
log.Any("mediaTaskId", item.MediaTaskId),
|
||||
log.Any("result", result))
|
||||
markFailedIfMediaSourceFailed(ctx, item)
|
||||
return watermarkSubmitFailed
|
||||
}
|
||||
}
|
||||
|
||||
func markFailedIfMediaSourceFailed(ctx context.Context, item *playletchaptermod.PlayletChapter) {
|
||||
if item == nil || item.MediaSourceId == "" {
|
||||
return
|
||||
}
|
||||
source, err := proxy.QueryMediaSource(item.MediaSourceId, pubdef.AppId, "")
|
||||
if err != nil {
|
||||
log.Info("playletwatermark:markFailedIfMediaSourceFailed 查询原片失败",
|
||||
log.Any("chapterId", item.Id),
|
||||
log.Any("mediaSourceId", item.MediaSourceId),
|
||||
log.E(err))
|
||||
return
|
||||
}
|
||||
if source == nil || source.Status != proxy.MediaSourceCompletedFail {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
if updateErr := playletchaptermod.UpdateOnePlayletChapter(ctx,
|
||||
bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.Id: item.Id,
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: bson.M{
|
||||
"$in": []playletchaptermod.WatermarkStatus{
|
||||
playletchaptermod.WatermarkStatusRegistered,
|
||||
playletchaptermod.WatermarkStatusProcessing,
|
||||
},
|
||||
},
|
||||
},
|
||||
bson.M{"$set": bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: playletchaptermod.WatermarkStatusFailed,
|
||||
playletchaptermod.FN.PlayletChapter.UpdatedAt: now,
|
||||
}}); updateErr != nil {
|
||||
log.Error("playletwatermark:markFailedIfMediaSourceFailed 更新章节失败",
|
||||
log.Any("chapterId", item.Id),
|
||||
log.E(updateErr))
|
||||
return
|
||||
}
|
||||
log.Info("playletwatermark:markFailedIfMediaSourceFailed 原片已失败，章节标为处理失败",
|
||||
log.Any("chapterId", item.Id),
|
||||
log.Any("mediaSourceId", item.MediaSourceId),
|
||||
log.Any("failReason", source.FailReason))
|
||||
}
|
||||
|
||||
// normalizeMediaFilePath 将相对/绝对 m3u8 地址规范化为 media 可识别的相对路径（不含 .m3u8）
|
||||
func normalizeMediaFilePath(raw string) (fieldPath string, version string) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return "", ""
|
||||
}
|
||||
version = mediaFileVersionV2
|
||||
if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return "", ""
|
||||
}
|
||||
raw = u.Path
|
||||
}
|
||||
raw = strings.TrimPrefix(raw, "/")
|
||||
if strings.HasPrefix(raw, "v1/") {
|
||||
version = mediaFileVersionV1
|
||||
}
|
||||
if strings.HasSuffix(strings.ToLower(raw), ".m3u8") {
|
||||
raw = strings.TrimSuffix(raw, ".m3u8")
|
||||
raw = strings.TrimSuffix(raw, ".M3U8")
|
||||
}
|
||||
raw = path.Clean(raw)
|
||||
if raw == "." || raw == "" {
|
||||
return "", ""
|
||||
}
|
||||
return raw, version
|
||||
}
|
||||
|
||||
// mediaWatermarkTitle 传给 media 的标题：短剧剧名（没有则用标题）+ "-" + 剧集标题
|
||||
func mediaWatermarkTitle(playlet *playletmod.Playlet, chapter *playletchaptermod.PlayletChapter) string {
|
||||
playletName := ""
|
||||
if playlet != nil {
|
||||
playletName = strings.TrimSpace(playlet.PlayName)
|
||||
if playletName == "" {
|
||||
playletName = strings.TrimSpace(playlet.Title)
|
||||
}
|
||||
}
|
||||
episodeTitle := ""
|
||||
if chapter != nil {
|
||||
episodeTitle = strings.TrimSpace(chapter.Title)
|
||||
}
|
||||
if playletName != "" && episodeTitle != "" {
|
||||
return playletName + "-" + episodeTitle
|
||||
}
|
||||
if playletName != "" {
|
||||
return playletName
|
||||
}
|
||||
return episodeTitle
|
||||
}
|
||||
|
||||
func playletForChapter(item *playletchaptermod.PlayletChapter) *playletmod.Playlet {
|
||||
if item == nil || item.PlayletId <= 0 {
|
||||
return nil
|
||||
}
|
||||
playlet, err := playletmod.FindOnePlaylet(context.Background(), bson.M{
|
||||
playletmod.FN.Playlet.Id: item.PlayletId,
|
||||
}, options.FindOne().SetProjection(bson.M{
|
||||
playletmod.FN.Playlet.PlayName: 1,
|
||||
playletmod.FN.Playlet.Title: 1,
|
||||
}))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return playlet
|
||||
}
|
||||
|
||||
func hasVideoWatermark(sysCfg *cfg.SysCfg) bool {
|
||||
if sysCfg == nil {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(sysCfg.WatermarkMedia) != "" ||
|
||||
strings.TrimSpace(sysCfg.WatermarkMediaTop) != "" ||
|
||||
strings.TrimSpace(sysCfg.WatermarkMediaTopRight) != "" ||
|
||||
strings.TrimSpace(sysCfg.WatermarkMediaBottomLeft) != ""
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package playletwatermark
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/srvdef"
|
||||
"videoweb/pub/models/playletchaptermod"
|
||||
"videoweb/pub/models/playletmod"
|
||||
"videoweb/pub/models/syncfilemod"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/pub/storage/proxy"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const (
|
||||
syncPageSize int64 = 1000
|
||||
syncOverlapWindow = 10 * time.Minute
|
||||
reconcileBatchSize int64 = 100
|
||||
)
|
||||
|
||||
var (
|
||||
syncRunning bool
|
||||
syncLock sync.Mutex
|
||||
)
|
||||
|
||||
// SyncCompletedChapters 定时从 media 拉取已完成水印任务，回写剧集地址与状态
|
||||
func SyncCompletedChapters() {
|
||||
syncLock.Lock()
|
||||
if syncRunning {
|
||||
syncLock.Unlock()
|
||||
return
|
||||
}
|
||||
syncRunning = true
|
||||
syncLock.Unlock()
|
||||
defer func() {
|
||||
syncLock.Lock()
|
||||
syncRunning = false
|
||||
syncLock.Unlock()
|
||||
}()
|
||||
|
||||
ctx := context.Background()
|
||||
log.Info("playletwatermark:SyncCompletedChapters begin")
|
||||
|
||||
record, err := ensureSyncRecord(ctx)
|
||||
if err != nil {
|
||||
log.Error("playletwatermark:SyncCompletedChapters 获取同步水印失败", log.E(err))
|
||||
return
|
||||
}
|
||||
|
||||
recordTime, syncOK := syncIncrementalCompletedChapters(ctx, record.RecordTime)
|
||||
if syncOK {
|
||||
if updateErr := syncfilemod.UpdateOneSyncRecord(ctx, bson.M{
|
||||
syncfilemod.FN.SyncRecord.Job: syncfilemod.PlayletWatermarkJob,
|
||||
}, bson.M{"$set": bson.M{
|
||||
syncfilemod.FN.SyncRecord.RecordTime: recordTime,
|
||||
syncfilemod.FN.SyncRecord.SuccessAt: time.Now(),
|
||||
syncfilemod.FN.SyncRecord.UpdatedAt: time.Now(),
|
||||
}}); updateErr != nil {
|
||||
log.Error("playletwatermark:SyncCompletedChapters 更新同步水印失败", log.E(updateErr))
|
||||
}
|
||||
}
|
||||
|
||||
reconcileProcessingChapters(ctx)
|
||||
log.Info("playletwatermark:SyncCompletedChapters end",
|
||||
log.Any("recordTime", recordTime),
|
||||
log.Any("syncOK", syncOK))
|
||||
}
|
||||
|
||||
// syncIncrementalCompletedChapters 按更新时间水印增量拉取。
|
||||
// 每次回退一段时间重复扫描，避免临界时间及同一时间戳分页造成漏数据；章节回写是幂等的。
|
||||
func syncIncrementalCompletedChapters(ctx context.Context, recordTime time.Time) (time.Time, bool) {
|
||||
cursor := recordTime
|
||||
if !cursor.IsZero() {
|
||||
cursor = cursor.Add(-syncOverlapWindow)
|
||||
}
|
||||
maxRecordTime := recordTime
|
||||
for {
|
||||
req := proxy.MediaTaskSyncReq{
|
||||
AppId: pubdef.AppId,
|
||||
LastUpdateTime: cursor,
|
||||
PageSize: syncPageSize,
|
||||
}
|
||||
result, syncErr := proxy.SyncMedia(&req, "")
|
||||
if syncErr != nil || result == nil || result.Code != http.StatusOK {
|
||||
log.Error("playletwatermark:SyncCompletedChapters SyncMedia 失败", log.Any("req", req), log.E(syncErr))
|
||||
return recordTime, false
|
||||
}
|
||||
pageMaxTime := cursor
|
||||
for _, media := range result.Data {
|
||||
if media.UpdatedAt.After(pageMaxTime) {
|
||||
pageMaxTime = media.UpdatedAt
|
||||
}
|
||||
if media.UpdatedAt.After(maxRecordTime) {
|
||||
maxRecordTime = media.UpdatedAt
|
||||
}
|
||||
applyCompletedMedia(ctx, media, "incremental")
|
||||
}
|
||||
if int64(len(result.Data)) < syncPageSize {
|
||||
break
|
||||
}
|
||||
if !pageMaxTime.After(cursor) {
|
||||
log.Error("playletwatermark:SyncCompletedChapters 增量分页水印未推进",
|
||||
log.Any("cursor", cursor),
|
||||
log.Any("count", len(result.Data)))
|
||||
return recordTime, false
|
||||
}
|
||||
cursor = pageMaxTime
|
||||
}
|
||||
return maxRecordTime, true
|
||||
}
|
||||
|
||||
// reconcileProcessingChapters 反向对账所有处理中章节。
|
||||
// 它不依赖增量水印，可补偿本地更新失败、时间戳临界和历史水印越过造成的漏回写。
|
||||
func reconcileProcessingChapters(ctx context.Context) {
|
||||
var lastID int64
|
||||
for {
|
||||
list, err := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.Id: bson.M{"$gt": lastID},
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: playletchaptermod.WatermarkStatusProcessing,
|
||||
playletchaptermod.FN.PlayletChapter.MediaSourceId: bson.M{"$gt": ""},
|
||||
}, options.Find().
|
||||
SetSort(bson.D{{Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1}}).
|
||||
SetLimit(reconcileBatchSize))
|
||||
if err != nil {
|
||||
log.Error("playletwatermark:reconcileProcessingChapters 查询章节失败", log.E(err))
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
sourceIDs := make([]string, 0, len(list))
|
||||
for _, item := range list {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
sourceIDs = append(sourceIDs, item.MediaSourceId)
|
||||
if item.Id > lastID {
|
||||
lastID = item.Id
|
||||
}
|
||||
}
|
||||
if len(sourceIDs) > 0 {
|
||||
result, queryErr := proxy.QueryMedias(&proxy.MediaQueryesReq{
|
||||
AppId: pubdef.AppId,
|
||||
MediaSourceIds: sourceIDs,
|
||||
}, "")
|
||||
if queryErr != nil || result == nil || result.Code != http.StatusOK {
|
||||
log.Error("playletwatermark:reconcileProcessingChapters 查询 media 任务失败",
|
||||
log.Any("sourceCount", len(sourceIDs)),
|
||||
log.Any("result", result),
|
||||
log.E(queryErr))
|
||||
return
|
||||
}
|
||||
for _, media := range result.Data {
|
||||
applyCompletedMedia(ctx, media, "reconcile")
|
||||
}
|
||||
}
|
||||
if int64(len(list)) < reconcileBatchSize {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func applyCompletedMedia(ctx context.Context, media proxy.MediaResp, source string) bool {
|
||||
sourceID := media.MediaSourceId.Hex()
|
||||
if sourceID == "" || sourceID == "000000000000000000000000" {
|
||||
log.Warn("playletwatermark:applyCompletedMedia mediaSourceId 无效",
|
||||
log.Any("mediaTaskId", media.Id.Hex()),
|
||||
log.Any("source", source))
|
||||
return false
|
||||
}
|
||||
fieldPath := strings.TrimSpace(media.FieldNameFs)
|
||||
if fieldPath == "" {
|
||||
log.Warn("playletwatermark:applyCompletedMedia 视频路径为空",
|
||||
log.Any("mediaSourceId", sourceID),
|
||||
log.Any("mediaTaskId", media.Id.Hex()),
|
||||
log.Any("source", source))
|
||||
return false
|
||||
}
|
||||
if !strings.HasSuffix(strings.ToLower(fieldPath), ".m3u8") {
|
||||
fieldPath += ".m3u8"
|
||||
}
|
||||
taskID := media.Id.Hex()
|
||||
setDoc := bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkVideoUrl: fieldPath,
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: playletchaptermod.WatermarkStatusDone,
|
||||
playletchaptermod.FN.PlayletChapter.MediaTaskId: taskID,
|
||||
playletchaptermod.FN.PlayletChapter.UpdatedAt: time.Now(),
|
||||
}
|
||||
if sourceName := strings.TrimSpace(media.SourceNameFs); sourceName != "" &&
|
||||
isOriginalPlayletBySourceID(ctx, sourceID) {
|
||||
// 原创才回填原片地址，供换水印/重置复用；非原创保留媒资原始 VideoUrl。
|
||||
setDoc[playletchaptermod.FN.PlayletChapter.VideoUrl] = sourceName
|
||||
}
|
||||
updateErr := playletchaptermod.UpdateOnePlayletChapter(ctx, bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.MediaSourceId: sourceID,
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkStatus: playletchaptermod.WatermarkStatusProcessing,
|
||||
}, bson.M{"$set": setDoc})
|
||||
if updateErr == nil {
|
||||
log.Info("playletwatermark:applyCompletedMedia 回写成功",
|
||||
log.Any("mediaSourceId", sourceID),
|
||||
log.Any("mediaTaskId", taskID),
|
||||
log.Any("watermarkVideoUrl", fieldPath),
|
||||
log.Any("source", source))
|
||||
return true
|
||||
}
|
||||
if updateErr.Code != srvdef.ErrNoData {
|
||||
log.Error("playletwatermark:applyCompletedMedia 更新章节失败",
|
||||
log.Any("mediaSourceId", sourceID),
|
||||
log.Any("source", source),
|
||||
log.E(updateErr))
|
||||
return false
|
||||
}
|
||||
|
||||
chapter, findErr := playletchaptermod.FindOnePlayletChapter(ctx, bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.MediaSourceId: sourceID,
|
||||
})
|
||||
if findErr != nil {
|
||||
if findErr.Code != srvdef.ErrNoData {
|
||||
log.Error("playletwatermark:applyCompletedMedia 查询未匹配章节失败",
|
||||
log.Any("mediaSourceId", sourceID),
|
||||
log.E(findErr))
|
||||
} else {
|
||||
log.Warn("playletwatermark:applyCompletedMedia 未找到对应章节",
|
||||
log.Any("mediaSourceId", sourceID),
|
||||
log.Any("mediaTaskId", taskID),
|
||||
log.Any("source", source))
|
||||
}
|
||||
return false
|
||||
}
|
||||
if chapter.WatermarkStatus != playletchaptermod.WatermarkStatusDone {
|
||||
log.Warn("playletwatermark:applyCompletedMedia 章节状态不允许回写",
|
||||
log.Any("chapterId", chapter.Id),
|
||||
log.Any("mediaSourceId", sourceID),
|
||||
log.Any("watermarkStatus", chapter.WatermarkStatus),
|
||||
log.Any("source", source))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isOriginalPlayletBySourceID(ctx context.Context, sourceID string) bool {
|
||||
chapter, err := playletchaptermod.FindOnePlayletChapter(ctx, bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.MediaSourceId: sourceID,
|
||||
}, options.FindOne().SetProjection(bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.PlayletId: 1,
|
||||
}))
|
||||
if err != nil || chapter == nil || chapter.PlayletId <= 0 {
|
||||
return false
|
||||
}
|
||||
playlet, err := playletmod.FindOnePlaylet(ctx, bson.M{
|
||||
playletmod.FN.Playlet.Id: chapter.PlayletId,
|
||||
}, options.FindOne().SetProjection(bson.M{
|
||||
playletmod.FN.Playlet.IsOriginal: 1,
|
||||
}))
|
||||
return err == nil && playlet != nil && playlet.IsOriginal
|
||||
}
|
||||
|
||||
func ensureSyncRecord(ctx context.Context) (*syncfilemod.SyncRecord, *srvdef.SrvErr) {
|
||||
filter := bson.M{syncfilemod.FN.SyncRecord.Job: syncfilemod.PlayletWatermarkJob}
|
||||
record, err := syncfilemod.FindOneSyncRecord(ctx, filter)
|
||||
if err == nil {
|
||||
return record, nil
|
||||
}
|
||||
if err.Code != srvdef.ObjectDayStatDoesNotExist {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now()
|
||||
record = &syncfilemod.SyncRecord{
|
||||
Job: syncfilemod.PlayletWatermarkJob,
|
||||
RecordTime: time.Time{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if insertErr := syncfilemod.InsertOneSyncRecord(ctx, record); insertErr != nil {
|
||||
return nil, insertErr
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package skd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/rand"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/web/service/srv_aicomment"
|
||||
"videoweb/web/service/srv_mediaresource"
|
||||
job "videoweb/web/skd/job/reportdata"
|
||||
seopushjob "videoweb/web/skd/job/seopush"
|
||||
"videoweb/web/skd/job/vipexpiry"
|
||||
"videoweb/web/skd/playletwatermark"
|
||||
"videoweb/web/skd/videosync"
|
||||
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
var (
|
||||
skdCron *cron.Cron
|
||||
)
|
||||
|
||||
func init() {
|
||||
skdCron = cron.New(cron.WithSeconds()) // 秒级定时任务
|
||||
}
|
||||
|
||||
func Start() {
|
||||
log.Info("skd start...")
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
_, _ = skdCron.AddFunc("0 0/2 * * * ?", func() {
|
||||
srv_aicomment.PollAiCommentResults(context.Background())
|
||||
})
|
||||
_, _ = skdCron.AddFunc("0 0/15 * * * ?", func() {
|
||||
srv_aicomment.RetryFailedPlaylets(context.Background())
|
||||
})
|
||||
|
||||
// 视频同步：每10分钟从媒资库增量同步到 video_information
|
||||
_, _ = skdCron.AddFunc("0 */10 * * * ?", func() {
|
||||
videosync.VideoSync()
|
||||
})
|
||||
|
||||
// 短剧水印：每5分钟提交最多10条未处理剧集到 media
|
||||
_, _ = skdCron.AddFunc("0 */5 * * * ?", func() {
|
||||
playletwatermark.SubmitPendingChapters()
|
||||
})
|
||||
// 短剧水印：每3分钟拉取 media 已完成任务并回写剧集
|
||||
_, _ = skdCron.AddFunc("0 */3 * * * ?", func() {
|
||||
playletwatermark.SyncCompletedChapters()
|
||||
})
|
||||
// 连载中短剧：每60分钟扫描一次并投递到 Redis 队列，由后台消费者逐条同步
|
||||
_, _ = skdCron.AddFunc(srv_mediaresource.PlayletSerialSyncCronSpec, func() {
|
||||
srv_mediaresource.EnqueueSerialPlayletSyncJobs()
|
||||
})
|
||||
// 历史短剧压缩封面：每5分钟扫描压缩封面为空的记录并投递队列
|
||||
_, _ = skdCron.AddFunc(srv_mediaresource.PlayletCoverCompressCronSpec, func() {
|
||||
srv_mediaresource.EnqueuePlayletCoverCompressJobs()
|
||||
})
|
||||
_, _ = skdCron.AddFunc(srv_mediaresource.AIContentBackflowCronSpec(), func() {
|
||||
go srv_mediaresource.ReportAIContentBackflowYesterday()
|
||||
})
|
||||
_, _ = skdCron.AddFunc(srv_mediaresource.AIContentBackflowTodayStatCronSpec(), func() {
|
||||
go srv_mediaresource.RefreshAIContentBackflowTodayStats()
|
||||
})
|
||||
srv_mediaresource.StartCGDJOriginalBackfillOnce(context.Background())
|
||||
go srv_mediaresource.RefreshAIContentBackflowTodayStats()
|
||||
|
||||
_, _ = skdCron.AddFunc("0/1 * * * * ?", func() { // 每秒执行一次批量上报
|
||||
job.Report()
|
||||
})
|
||||
|
||||
// 每天凌晨 02:00 重置 VIP 已过期用户的 VipType
|
||||
_, _ = skdCron.AddFunc("0 0 2 * * ?", func() {
|
||||
vipexpiry.ResetExpiredVip()
|
||||
})
|
||||
|
||||
// 搜索引擎推送兜底：补推已上架短剧并重试失败任务。
|
||||
// 开关关闭或非正式环境不会真正对外提交，详见 srv_seopush.seoPushEnabled
|
||||
_, _ = skdCron.AddFunc("0/30 * * * * ?", func() {
|
||||
seopushjob.Run()
|
||||
})
|
||||
|
||||
//if webg.Conf.LsjMediaConf.ApiUrl != "" {
|
||||
// _, _ = skdCron.AddFunc("0 30 */2 * * ?", func() {
|
||||
// transcodejob.TranscodeVideoByWatchTimes()
|
||||
// })
|
||||
// _, _ = skdCron.AddFunc("0 */5 * * * ?", func() {
|
||||
// transcodejob.CheckTranscodingResults()
|
||||
// })
|
||||
//}
|
||||
|
||||
skdCron.Start()
|
||||
}
|
||||
|
||||
func Stop() {
|
||||
log.Info("srv_datinggirl:updater Stop...")
|
||||
defer log.Info("srv_datinggirl: updater Stop!!!")
|
||||
srv_mediaresource.CloseAIContentBackflowClient()
|
||||
skdCron.Stop()
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package videosync
|
||||
|
||||
import (
|
||||
"time"
|
||||
"videoweb/pub/models/videoinformationmod"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
)
|
||||
|
||||
type MediaTaskSyncReq struct {
|
||||
AppId int64 `bson:"appId" json:"appId"` // APPID
|
||||
LastUpdateTime time.Time `bson:"lastUpdateTime" json:"lastUpdateTime"` // 最后更新时间
|
||||
PageSize int64 `bson:"pageSize" json:"pageSize"`
|
||||
}
|
||||
|
||||
type MediaTaskSyncResp struct {
|
||||
Code int `json:"code,omitempty"` //
|
||||
Data []MediaTaskInfo `json:"data,omitempty"` //
|
||||
Msg string `json:"msg,omitempty"` //
|
||||
}
|
||||
|
||||
// @table:media
|
||||
type MediaTaskInfo struct {
|
||||
Id primitive.ObjectID `bson:"_id,omitempty" json:"id"`
|
||||
MediaSourceId primitive.ObjectID `bson:"mediaSourceId,omitempty" json:"mediaSourceId"` // 原始Id
|
||||
VideoTime int32 `bson:"videoTime,omitempty" json:"videoTime"` // 影片长度
|
||||
Size int64 `bson:"size,omitempty" json:"size"` // 文件size，单位byte
|
||||
AppId int64 `bson:"appId,omitempty" json:"appId"` // 应用列表
|
||||
CoverImg string `bson:"coverImg,omitempty" json:"coverImg"` // 封面
|
||||
CoverImgNew string `bson:"coverImgNew,omitempty" json:"coverImgNew"` // 动态封面
|
||||
UnencryptedImage string `bson:"unencryptedImage,omitempty" json:"unencryptedImage"` // 不加密封面
|
||||
Title string `bson:"title,omitempty" json:"title"` // 标题
|
||||
Publish string `bson:"publish,omitempty" json:"publish"` // 视频发布者
|
||||
Remark string `bson:"remark,omitempty" json:"remark"` // 备注
|
||||
Keyworld []string `bson:"keyworld,omitempty" json:"keyworld"` // 关键字
|
||||
|
||||
Nodes []videoinformationmod.Node `bson:"nodes" json:"nodes"` // 节点
|
||||
FirstTag string `bson:"firstTag,omitempty" json:"firstTag"` // 一级标签
|
||||
SecondTags []string `bson:"secondTags,omitempty" json:"secondTags"` // 二级标签标签
|
||||
Status string `bson:"status,omitempty" json:"status"` // 当前状态
|
||||
Desc string `bson:"desc,omitempty" json:"desc"` // 详细描述
|
||||
TaskStartTime time.Time `bson:"taskStartTime,omitempty" json:"taskStartTime"` // 任务开始时间
|
||||
SourceNameFs string `bson:"sourceNameFs,omitempty" json:"sourceNameFs"` // 远程源文件
|
||||
IsDelLocalDir bool `bson:"isDelLocalDir,omitempty" json:"isDelLocalDir"` // 是否删除本机文件
|
||||
WatermarkImage string `json:"watermarkImage" bson:"watermarkImage"` // 图片水印文件
|
||||
WatermarkMedia string `json:"watermarkMedia" bson:"watermarkMedia"` // 视频水印文件
|
||||
DownloadRetry int `bson:"downloadRetry,omitempty" json:"downloadRetry"` // 下载重试次数
|
||||
WatermarkRetry int `bson:"watermarkRetry" json:"watermarkRetry"` // 水印重试次数
|
||||
CompressRetry int `bson:"compressRetry" json:"compressRetry"` // 压缩重试次数
|
||||
SegmentRetry int `bson:"segmentRetry" json:"segmentRetry"` // 切片重试次数
|
||||
LinodeRetry int `bson:"linodeRetry" json:"linodeRetry"` // 上传重试次数
|
||||
|
||||
// 切片相关信息
|
||||
Filename string `bson:"filename,omitempty" json:"filename"` // 文件名称
|
||||
FieldNameFs string `bson:"fieldNameFs,omitempty" json:"fieldNameFs"` // 文件名称
|
||||
TempFieldNameFs string `bson:"tempFieldNameFs,omitempty" json:"tempFieldNameFs"` // 临时存储
|
||||
Ext string `bson:"ext,omitempty" json:"ext"` // 视频后缀
|
||||
UnencryptedPreMp4 string `bson:"unencryptedPreMp4,omitempty" json:"unencryptedPreMp4"` // 不加密的预览MP4
|
||||
TaskID string `bson:"taskId,omitempty" json:"taskId"` // 项目组传来的唯一标示
|
||||
Width int64 `bson:"width,omitempty" json:"width"` // 分辨率宽
|
||||
Height int64 `bson:"height,omitempty" json:"height"` // 分辨率高
|
||||
Ratio float64 `bson:"ratio,omitempty" json:"ratio"` // 宽高比
|
||||
FrameRate string `bson:"frameRate,omitempty" json:"frameRate"` // 视频贞数
|
||||
SyBitRate string `bson:"syBitRate,omitempty" json:"syBitRate"` // 音频bit率
|
||||
VidBitRate string `bson:"vidBitRate,omitempty" json:"vidBitRate"` // 视频bit率
|
||||
CreatedAt time.Time `bson:"createdAt,omitempty" json:"createdAt"` // 创建时间
|
||||
UpdatedAt time.Time `bson:"updatedAt,omitempty" json:"updatedAt"` // 更新时间
|
||||
ProxyUpload *int32 `bson:"proxyUpload,omitempty" json:"proxyUpload"` // 代理上传
|
||||
Retry int `bson:"retry" json:"retry"` // 重试次数 默认为零 最大重试3次
|
||||
NotifySignal bool `bson:"notifySignal" json:"notifySignal"` //中止通知的信号 false 可以发送通知,true 中止发送通知
|
||||
|
||||
InfoTsName string `bson:"infoTsName" json:"infoTsName"` // 从该ts中获取视频分辨率等信息
|
||||
SyncFlag int64 `bson:"syncFlag" json:"syncFlag"` // 同步状态
|
||||
SrcKey string `bson:"srcKey,omitempty" json:"srcKey"` // 原始Key
|
||||
UploadUserName string `bson:"uploadUserName" json:"uploadUserName,omitempty"` // 上传用户名
|
||||
OptionAccount string `bson:"optionAccount,omitempty" json:"optionAccount"` // 最后操作人
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package videosync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
nethttp "net/http"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/srvdef"
|
||||
"videoweb/pub/models/syncfilemod"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/pub/storage/proxy"
|
||||
"videoweb/web/service/srv_videoinfo"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
)
|
||||
|
||||
/*
|
||||
1.小说采集后上传媒资库
|
||||
2.业务后台记录最新更新章节ID(数据ID自增)
|
||||
3.查询对应小说ID是否存在,如果不存在需要先同步媒资库小说信息，再更新小说
|
||||
注意：业务后台和媒资库 存储的小说ID和章节ID需一致， 资源以媒资库为准， 不单独更新业务后台小说
|
||||
*/
|
||||
|
||||
var (
|
||||
isRunning = false
|
||||
lock sync.Mutex
|
||||
)
|
||||
|
||||
func getIsRunning() bool {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
return isRunning
|
||||
}
|
||||
func setRunning(b bool) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
isRunning = b
|
||||
}
|
||||
func VideoSync() {
|
||||
isRunning = getIsRunning()
|
||||
if isRunning {
|
||||
return
|
||||
}
|
||||
setRunning(true)
|
||||
ctx := context.Background()
|
||||
log.Info("VideoSync begin")
|
||||
pageSize := int64(200)
|
||||
recordFilter := bson.M{
|
||||
syncfilemod.FN.SyncRecord.Job: syncfilemod.VideoJob,
|
||||
}
|
||||
recordInfo, serr := syncfilemod.FindOneSyncRecord(ctx, recordFilter)
|
||||
if serr != nil {
|
||||
if serr.Code == srvdef.ObjectDayStatDoesNotExist {
|
||||
log.Info("VideoSync init record", log.Any("job", syncfilemod.VideoJob))
|
||||
syncfilemod.InsertOneSyncRecord(ctx, &syncfilemod.SyncRecord{
|
||||
ID: primitive.NewObjectID(),
|
||||
Job: syncfilemod.VideoJob,
|
||||
RecordId: 1,
|
||||
RecordTime: time.Time{},
|
||||
SuccessAt: time.Time{},
|
||||
CreatedAt: time.Time{},
|
||||
UpdatedAt: time.Time{},
|
||||
})
|
||||
} else {
|
||||
log.Error("VideoSync FindOneSyncRecord failed", log.Any("recordFilter", recordFilter), log.E(serr))
|
||||
}
|
||||
setRunning(false)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
setRunning(false)
|
||||
// 更新同步记录
|
||||
set := bson.M{
|
||||
"$set": bson.M{
|
||||
syncfilemod.FN.SyncRecord.RecordTime: recordInfo.RecordTime,
|
||||
syncfilemod.FN.SyncRecord.SuccessAt: time.Now(),
|
||||
},
|
||||
}
|
||||
_ = syncfilemod.UpdateOneSyncRecord(ctx, recordFilter, set)
|
||||
}()
|
||||
// 循环同步
|
||||
for {
|
||||
// 查询接口
|
||||
req := proxy.MediaTaskSyncReq{
|
||||
AppId: pubdef.AppId,
|
||||
LastUpdateTime: recordInfo.RecordTime,
|
||||
PageSize: pageSize,
|
||||
}
|
||||
result, err := proxy.SyncMedia(&req, "")
|
||||
if err != nil || result.Code != nethttp.StatusOK {
|
||||
log.Error("VideoSync SyncMedia", log.Any("req", req), log.E(err))
|
||||
return
|
||||
}
|
||||
|
||||
// 处理
|
||||
for _, media := range result.Data {
|
||||
// 先推进水位，确保单条失败后下次不会卡在同一位置
|
||||
if media.UpdatedAt.After(recordInfo.RecordTime) {
|
||||
recordInfo.RecordTime = media.UpdatedAt
|
||||
}
|
||||
if srv_videoinfo.IsPlayletMedia(ctx, media) {
|
||||
log.Info("VideoSync skip playlet media", log.Any("mediaId", media.Id.Hex()), log.Any("mediaSourceId", media.MediaSourceId.Hex()))
|
||||
continue
|
||||
}
|
||||
se := srv_videoinfo.SyncOneVideo(ctx, media)
|
||||
if se != nil {
|
||||
log.Error("VideoSync SyncOneVideo", log.Any("media", media), log.E(se))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if len(result.Data) < int(pageSize) {
|
||||
log.Info("VideoSync success", log.Any("RecordTime", recordInfo.RecordTime))
|
||||
break
|
||||
}
|
||||
}
|
||||
log.Info("VideoSync success", log.Any("RecordTime", recordInfo.RecordTime))
|
||||
}
|
||||
|
||||
// SyncUnencryptedFields 全量补齐历史资源的 UnencryptedImage / UnencryptedPreMp4 字段。
|
||||
// 从头分页拉取媒资库，仅在源端有值时写回，本地不存在或字段已有值不覆盖。
|
||||
// 独立于增量同步的 SyncRecord，用于一次性修复；可安全重复执行。
|
||||
func SyncUnencryptedFields() {
|
||||
ctx := context.Background()
|
||||
log.Info("SyncUnencryptedFields begin")
|
||||
pageSize := int64(200)
|
||||
lastUpdateTime := time.Time{}
|
||||
for {
|
||||
req := proxy.MediaTaskSyncReq{
|
||||
AppId: pubdef.AppId,
|
||||
LastUpdateTime: lastUpdateTime,
|
||||
PageSize: pageSize,
|
||||
}
|
||||
result, err := proxy.SyncMedia(&req, "")
|
||||
if err != nil || result.Code != nethttp.StatusOK {
|
||||
log.Error("SyncUnencryptedFields SyncMedia", log.Any("req", req), log.E(err))
|
||||
return
|
||||
}
|
||||
for _, media := range result.Data {
|
||||
// 先推进时间戳，保证 continue 也不阻塞分页
|
||||
if media.UpdatedAt.After(lastUpdateTime) {
|
||||
lastUpdateTime = media.UpdatedAt
|
||||
}
|
||||
se := srv_videoinfo.SyncUnencryptedOne(ctx, media)
|
||||
if se != nil && !errors.Is(se.Code, srvdef.ErrNoData) {
|
||||
log.Warn("SyncUnencryptedFields SyncUnencryptedOne", log.Any("mediaId", media.Id.Hex()), log.E(se))
|
||||
}
|
||||
}
|
||||
if len(result.Data) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
log.Info("SyncUnencryptedFields success", log.Any("lastUpdateTime", lastUpdateTime))
|
||||
}
|
||||
Reference in New Issue
Block a user