package srv_mediaresource import ( "context" "fmt" "net/url" "path" "strconv" "strings" "sync" "time" "videoweb/pub/pubdef" "videoweb/pub/common/log" "videoweb/pub/common/srvdef" aidatacenter "videoweb/pub/data_center" "videoweb/pub/models/playletbackflowstatmod" "videoweb/pub/models/playletchaptermod" "videoweb/pub/models/playletmod" "videoweb/web/webg" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo/options" ) const ( // defaultAIContentBackflowPageSize 每页扫描原创短剧数量,避免一次性加载过多 Mongo 数据。 defaultAIContentBackflowPageSize = int64(200) // defaultAIContentBackflowBatchSize 每批提交给内容中台的记录数。 defaultAIContentBackflowBatchSize = int64(500) // maxAIContentBackflowBatchSize 内容中台单批最多允许 1000 条。 maxAIContentBackflowBatchSize = int64(1000) // defaultAIContentBackflowReportTimeout 单批同步上报的最长等待时间。 defaultAIContentBackflowReportTimeout = 60 * time.Second ) var ( // aiContentBackflowPlatformBaseURL 覆盖短剧播放页域名;App-only 产品为空时 platform_url 回退到 video_url。 aiContentBackflowPlatformBaseURL = "" // aiContentBackflowVideoBaseURL 覆盖视频回源域名;空则使用 dataCenterBackflow.selfSourceBaseUrl。 aiContentBackflowVideoBaseURL = "" // aiContentBackflowPageSize 运行时扫描分页大小,非法值回退默认值。 aiContentBackflowPageSize = int64(200) // aiContentBackflowBatchSize 运行时上报批量大小,非法值回退默认值。 aiContentBackflowBatchSize = int64(500) // aiContentBackflowCronSpec 运行时 cron 表达式,空值回退每天 02:30。 aiContentBackflowCronSpec = "" ) // aiContentBackflowStats 记录一次任务的扫描、构造和跳过数量。 type aiContentBackflowStats struct { DataDate string // 数据日期,格式 DD/MM/YYYY Scanned int64 // 扫描到的原创短剧数 Built int64 // 成功构造的回流记录数 Stored int64 // 已写入本地统计表的记录数 Sent int64 // 已发送给内容中台的记录数 Batches int64 // 发送批次数 Reported int64 // 已标记上报成功的记录数 Failed int64 // 已标记上报失败的记录数 Skipped int64 // 因数据缺口跳过的短剧数 MissingChapter int64 // 缺少可用章节的数量 MissingVideo int64 // 缺少视频地址的数量 MissingMediaContent int64 // 第一集缺少媒资章节 ID 的数量 FuturePublish int64 // 上线日期晚于统计日期的数量 } // aiContentBackflowChapters 一部短剧回流需要的章节:第一集,以及按集数最早的付费集。 type aiContentBackflowChapters struct { First *playletchaptermod.PlayletChapter FirstPaid *playletchaptermod.PlayletChapter } // aiContentBackflowStore 抽象数据读写和发送函数,方便单测注入假实现。 type aiContentBackflowStore struct { findPlaylets func(context.Context, bson.M, *options.FindOptions) (playletmod.PlayletList, *srvdef.SrvErr) findChapters func(context.Context, []int64) (map[int64]*aiContentBackflowChapters, *srvdef.SrvErr) } // aiContentBackflowClient 复用 SDK client,避免每次定时任务重复创建后台队列。 var aiContentBackflowClient = struct { sync.Mutex client *aidatacenter.Client apiKey string endpoint string }{} // AIContentBackflowCronSpec 返回原创短剧数据回流的每日调度时间。 func AIContentBackflowCronSpec() string { if spec := strings.TrimSpace(aiContentBackflowCronSpec); spec != "" { return spec } return "0 30 2 * * ?" } // AIContentBackflowTodayStatCronSpec 返回当天统计表小时级刷新时间。 func AIContentBackflowTodayStatCronSpec() string { return "0 0 * * * ?" } // ReportAIContentBackflowYesterday 上报昨日原创短剧数据。 // // 内容中台日记录按自然日覆盖更新,重复执行同一天不会累加。 func ReportAIContentBackflowYesterday() { loc, err := time.LoadLocation("Asia/Shanghai") if err != nil { loc = time.Local } targetDate := time.Now().In(loc).AddDate(0, 0, -1) stats, serr := reportAIContentBackflow(context.Background(), targetDate) if serr != nil { log.Error("AI 内容中台原创短剧数据回流失败", log.Any("err", serr), log.Any("stats", stats)) return } if stats != nil { log.Info("AI 内容中台原创短剧数据回流完成", log.Any("stats", stats)) } } // RefreshAIContentBackflowTodayStats 刷新当天原创短剧回流统计,便于上线后观察数据。 func RefreshAIContentBackflowTodayStats() { loc, err := time.LoadLocation("Asia/Shanghai") if err != nil { loc = time.Local } targetDate := time.Now().In(loc) stats, serr := refreshAIContentBackflowStats(context.Background(), targetDate) if serr != nil { log.Error("AI 内容中台原创短剧当天统计刷新失败", log.Any("err", serr), log.Any("stats", stats)) return } log.Info("AI 内容中台原创短剧当天统计刷新完成", log.Any("stats", stats)) } // CloseAIContentBackflowClient 关闭复用的内容中台 SDK client。 func CloseAIContentBackflowClient() { aiContentBackflowClient.Lock() client := aiContentBackflowClient.client aiContentBackflowClient.client = nil aiContentBackflowClient.apiKey = "" aiContentBackflowClient.endpoint = "" aiContentBackflowClient.Unlock() if client != nil { client.Close() } } // reportAIContentBackflow 从数据回流配置读取 API Key;空 key 表示未启用真实回流。 func reportAIContentBackflow(ctx context.Context, targetDate time.Time) (*aiContentBackflowStats, *srvdef.SrvErr) { apiKey := strings.TrimSpace(webg.Conf.DataCenterBackflow.Key) if apiKey == "" { log.Info("AI 内容中台原创短剧数据回流未启用", log.Any("dataDate", formatAIContentBackflowDate(targetDate)), log.Any("reason", "dataCenterBackflow.key empty"), ) return nil, nil } stats, serr := ensureAIContentBackflowStats(ctx, targetDate) if serr != nil { return stats, serr } client, serr := getAIContentBackflowClient(apiKey) if serr != nil { return stats, serr } reportStats, serr := reportAIContentBackflowStats(ctx, targetDate, client) if reportStats != nil { mergeAIContentBackflowStats(stats, *reportStats) } return stats, serr } // refreshAIContentBackflowStats 计算指定日期数据并写入本地统计表,不依赖内容中台 API Key。 func refreshAIContentBackflowStats(ctx context.Context, targetDate time.Time) (*aiContentBackflowStats, *srvdef.SrvErr) { return refreshAIContentBackflowStatsWithStore(ctx, targetDate, aiContentBackflowStore{ findPlaylets: findOriginalPlayletsForBackflow, findChapters: findFirstPlayletChaptersForBackflow, }) } // getAIContentBackflowClient 创建或复用 data_center SDK client。 func getAIContentBackflowClient(apiKey string) (*aidatacenter.Client, *srvdef.SrvErr) { if !strings.HasPrefix(apiKey, "ld_") { return nil, srvdef.New(srvdef.Failure, "AI内容中台API Key格式错误", "AI内容中台API Key格式错误") } aiContentBackflowClient.Lock() defer aiContentBackflowClient.Unlock() if aiContentBackflowClient.client != nil && aiContentBackflowClient.apiKey == apiKey { return aiContentBackflowClient.client, nil } if aiContentBackflowClient.client != nil { aiContentBackflowClient.client.Close() } client := aidatacenter.New(apiKey) endpoint := client.Endpoint client.OnError = func(err error) { log.Error("AI 内容中台原创短剧异步上报失败", log.Any("endpoint", endpoint), log.Any("companyPlatform", pubdef.CompanyPlatform), log.E(err), ) } log.Info("AI 内容中台原创短剧数据回流客户端初始化", log.Any("endpoint", endpoint), log.Any("companyPlatform", pubdef.CompanyPlatform), ) aiContentBackflowClient.client = client aiContentBackflowClient.apiKey = apiKey aiContentBackflowClient.endpoint = endpoint return client, nil } // refreshAIContentBackflowStatsWithStore 分页扫描原创短剧并写入本地日统计表。 func refreshAIContentBackflowStatsWithStore(ctx context.Context, targetDate time.Time, store aiContentBackflowStore) (*aiContentBackflowStats, *srvdef.SrvErr) { if serr := validateAIContentBackflowRuntime(); serr != nil { return nil, serr } pageSize := normalizeAIContentBackflowPageSize(aiContentBackflowPageSize) stats := &aiContentBackflowStats{DataDate: formatAIContentBackflowDate(targetDate)} log.Info("AI 内容中台原创短剧日统计刷新开始", log.Any("dataDate", stats.DataDate), log.Any("companyPlatform", pubdef.CompanyPlatform), log.Any("pageSize", pageSize), log.Any("videoBaseURL", resolveAIContentBackflowVideoBaseURL()), ) for lastID := int64(0); ; { list, serr := store.findPlaylets(ctx, originalPlayletBackflowFilter(lastID), originalPlayletBackflowFindOptions(pageSize)) if serr != nil { return stats, serr } if len(list) == 0 { break } stats.Scanned += int64(len(list)) docs, pageStats, nextID, serr := buildAIContentBackflowDailyStats(ctx, targetDate, list, store.findChapters) mergeAIContentBackflowStats(stats, pageStats) if serr != nil { return stats, serr } if serr := playletbackflowstatmod.UpsertManyPlayletBackflowDailyStats(ctx, docs); serr != nil { return stats, serr } stats.Stored += int64(len(docs)) log.Info("AI 内容中台原创短剧日统计分页刷新完成", log.Any("dataDate", stats.DataDate), log.Any("lastID", lastID), log.Any("nextID", nextID), log.Any("scanned", len(list)), log.Any("built", pageStats.Built), log.Any("stored", len(docs)), log.Any("skipped", pageStats.Skipped), log.Any("missingChapter", pageStats.MissingChapter), log.Any("missingVideo", pageStats.MissingVideo), log.Any("missingMediaContent", pageStats.MissingMediaContent), log.Any("futurePublish", pageStats.FuturePublish), ) if int64(len(list)) < pageSize || nextID <= lastID { break } lastID = nextID } return stats, nil } // ensureAIContentBackflowStats 确保指定日期已有本地统计,避免每日上报重复扫描。 func ensureAIContentBackflowStats(ctx context.Context, targetDate time.Time) (*aiContentBackflowStats, *srvdef.SrvErr) { dataDate := formatAIContentBackflowDate(targetDate) total, serr := playletbackflowstatmod.CountPlayletBackflowDailyStats(ctx, bson.M{ playletbackflowstatmod.FN.PlayletBackflowDailyStat.DataDate: dataDate, }) if serr != nil { return nil, serr } success, serr := playletbackflowstatmod.CountPlayletBackflowDailyStats(ctx, bson.M{ playletbackflowstatmod.FN.PlayletBackflowDailyStat.DataDate: dataDate, playletbackflowstatmod.FN.PlayletBackflowDailyStat.ReportStatus: playletbackflowstatmod.ReportStatusSuccess, }) if serr != nil { return nil, serr } if total > 0 && total == success { stats := &aiContentBackflowStats{DataDate: dataDate, Stored: total} log.Info("AI 内容中台原创短剧日统计已全部上报成功", log.Any("dataDate", stats.DataDate), log.Any("count", total), ) return stats, nil } log.Info("AI 内容中台原创短剧上报前刷新日统计", log.Any("dataDate", dataDate), log.Any("total", total), log.Any("success", success), ) return refreshAIContentBackflowStats(ctx, targetDate) } // reportAIContentBackflowStats 从本地统计表读取待上报数据,同步发送后标记状态。 func reportAIContentBackflowStats(ctx context.Context, targetDate time.Time, client *aidatacenter.Client) (*aiContentBackflowStats, *srvdef.SrvErr) { batchSize := normalizeAIContentBackflowBatchSize(aiContentBackflowBatchSize) stats := &aiContentBackflowStats{DataDate: formatAIContentBackflowDate(targetDate)} log.Info("AI 内容中台原创短剧统计上报开始", log.Any("dataDate", stats.DataDate), log.Any("batchSize", batchSize), ) for lastID, hasLastID := primitive.NilObjectID, false; ; { filter := bson.M{ playletbackflowstatmod.FN.PlayletBackflowDailyStat.DataDate: stats.DataDate, playletbackflowstatmod.FN.PlayletBackflowDailyStat.ReportStatus: bson.M{ "$ne": playletbackflowstatmod.ReportStatusSuccess, }, } if hasLastID { filter[playletbackflowstatmod.FN.PlayletBackflowDailyStat.Id] = bson.M{"$gt": lastID} } list, serr := playletbackflowstatmod.FindManyPlayletBackflowDailyStats(ctx, filter, options.Find(). SetSort(bson.D{{Key: playletbackflowstatmod.FN.PlayletBackflowDailyStat.Id, Value: 1}}). SetLimit(batchSize)) if serr != nil { return stats, serr } if len(list) == 0 { break } records := make([]aidatacenter.Record, 0, len(list)) ids := make([]primitive.ObjectID, 0, len(list)) for _, item := range list { if item == nil || item.Id.IsZero() { continue } lastID = item.Id hasLastID = true if !shouldReportAIContentBackflowStat(item) { stats.Skipped++ stats.FuturePublish++ log.Warn("AI 内容中台原创短剧统计跳过未来上线快照", log.Any("dataDate", item.DataDate), log.Any("playletId", item.PlayletId), log.Any("publishedData", item.PublishedData), ) continue } if item.MediaContentId <= 0 || item.ChapterId <= 0 { stats.Skipped++ stats.MissingMediaContent++ log.Warn("AI 内容中台原创短剧统计跳过缺少剧集标识的快照", log.Any("dataDate", item.DataDate), log.Any("playletId", item.PlayletId), log.Any("chapterId", item.ChapterId), log.Any("mediaContentId", item.MediaContentId), ) continue } records = append(records, makeAIContentBackflowRecordFromStat(item)) ids = append(ids, item.Id) } if len(records) == 0 { if int64(len(list)) < batchSize { break } continue } if len(ids) == 0 { break } sendCtx, cancel := context.WithTimeout(ctx, defaultAIContentBackflowReportTimeout) _, err := client.Send(sendCtx, records...) cancel() if err != nil { reportErr := err.Error() if serr := playletbackflowstatmod.MarkPlayletBackflowDailyStatsFailed(ctx, ids, reportErr); serr != nil { return stats, serr } stats.Failed += int64(len(ids)) log.Error("AI 内容中台原创短剧统计批次上报失败", log.Any("dataDate", stats.DataDate), log.Any("batchSize", len(records)), log.Any("ids", ids), log.E(err), ) return stats, srvdef.New(srvdef.Failure, "AI内容中台原创短剧上报失败", "AI内容中台原创短剧上报失败") } reportAt := time.Now() if serr := playletbackflowstatmod.MarkPlayletBackflowDailyStatsReported(ctx, ids, reportAt); serr != nil { return stats, serr } stats.Sent += int64(len(records)) stats.Reported += int64(len(ids)) stats.Batches++ log.Info("AI 内容中台原创短剧统计批次上报成功", log.Any("dataDate", stats.DataDate), log.Any("batchNo", stats.Batches), log.Any("batchSize", len(records)), log.Any("sent", stats.Sent), ) if int64(len(list)) < batchSize { break } } return stats, nil } // validateAIContentBackflowRuntime 校验代码内置口径是否能生成中台必填字段。 func validateAIContentBackflowRuntime() *srvdef.SrvErr { if strings.TrimSpace(pubdef.CompanyPlatform) == "" { return srvdef.New(srvdef.Failure, "AI内容中台产品名称未配置", "AI内容中台产品名称未配置") } if resolveAIContentBackflowVideoBaseURL() == "" { return srvdef.New(srvdef.Failure, "AI内容中台视频回源地址未配置", "AI内容中台视频回源地址未配置") } return nil } // normalizeAIContentBackflowPageSize 修正扫描分页大小。 func normalizeAIContentBackflowPageSize(v int64) int64 { if v <= 0 { return defaultAIContentBackflowPageSize } return v } // normalizeAIContentBackflowBatchSize 修正上报批量大小,并限制在中台上限内。 func normalizeAIContentBackflowBatchSize(v int64) int64 { if v <= 0 { return defaultAIContentBackflowBatchSize } if v > maxAIContentBackflowBatchSize { return maxAIContentBackflowBatchSize } return v } // originalPlayletBackflowFilter 只选择已标记媒资来源且已上架的短剧。 func originalPlayletBackflowFilter(lastID int64) bson.M { filter := bson.M{ playletmod.FN.Playlet.MediaBusiness: bson.M{"$gt": ""}, playletmod.FN.Playlet.Status: playletmod.StatusUp, } if lastID > 0 { filter[playletmod.FN.Playlet.Id] = bson.M{"$gt": lastID} } return filter } // originalPlayletBackflowFindOptions 按 ID 游标分页,只读取组装回流记录需要的字段。 func originalPlayletBackflowFindOptions(pageSize int64) *options.FindOptions { return options.Find(). SetSort(bson.D{{Key: playletmod.FN.Playlet.Id, Value: 1}}). SetLimit(pageSize). SetProjection(bson.M{ playletmod.FN.Playlet.Id: 1, playletmod.FN.Playlet.MediaCenterId: 1, playletmod.FN.Playlet.Title: 1, playletmod.FN.Playlet.Status: 1, playletmod.FN.Playlet.OnlineAt: 1, playletmod.FN.Playlet.MediaCreatedAt: 1, playletmod.FN.Playlet.CreatedAt: 1, playletmod.FN.Playlet.CountBrowse: 1, playletmod.FN.Playlet.CountLike: 1, playletmod.FN.Playlet.CountComment: 1, playletmod.FN.Playlet.TotalEpisode: 1, }) } // findOriginalPlayletsForBackflow 查询待回流原创短剧。 func findOriginalPlayletsForBackflow(ctx context.Context, filter bson.M, opts *options.FindOptions) (playletmod.PlayletList, *srvdef.SrvErr) { return playletmod.FindManyPlaylet(ctx, filter, opts) } // findFirstPlayletChaptersForBackflow 取每部短剧的第一集,以及按集数最早的付费集。 func findFirstPlayletChaptersForBackflow(ctx context.Context, playletIDs []int64) (map[int64]*aiContentBackflowChapters, *srvdef.SrvErr) { if len(playletIDs) == 0 { return map[int64]*aiContentBackflowChapters{}, nil } list, serr := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{ playletchaptermod.FN.PlayletChapter.PlayletId: bson.M{"$in": playletIDs}, }, options.Find(). SetSort(bson.D{ {Key: playletchaptermod.FN.PlayletChapter.PlayletId, Value: 1}, {Key: playletchaptermod.FN.PlayletChapter.CurrentEpisode, Value: 1}, {Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1}, }). SetProjection(bson.M{ playletchaptermod.FN.PlayletChapter.Id: 1, playletchaptermod.FN.PlayletChapter.MediaContentId: 1, playletchaptermod.FN.PlayletChapter.PlayletId: 1, playletchaptermod.FN.PlayletChapter.CurrentEpisode: 1, playletchaptermod.FN.PlayletChapter.VideoUrl: 1, playletchaptermod.FN.PlayletChapter.CountBrowse: 1, playletchaptermod.FN.PlayletChapter.CountLike: 1, playletchaptermod.FN.PlayletChapter.CountComment: 1, playletchaptermod.FN.PlayletChapter.CountBuy: 1, playletchaptermod.FN.PlayletChapter.PayType: 1, })) if serr != nil { return nil, serr } result := make(map[int64]*aiContentBackflowChapters, len(playletIDs)) for _, chapter := range list { if chapter == nil { continue } item := result[chapter.PlayletId] if item == nil { item = &aiContentBackflowChapters{First: chapter} result[chapter.PlayletId] = item } if item.FirstPaid == nil && isPaidPlayletChapter(chapter) { item.FirstPaid = chapter } } return result, nil } // isPaidPlayletChapter 金币或 VIP 剧集需要解锁后才能完整观看。 func isPaidPlayletChapter(chapter *playletchaptermod.PlayletChapter) bool { if chapter == nil { return false } return chapter.PayType == playletchaptermod.PayTypeCoin || chapter.PayType == playletchaptermod.PayTypeVip } // buildAIContentBackflowRecords 把短剧模型转换成内容中台 records。 func buildAIContentBackflowRecords( ctx context.Context, targetDate time.Time, playlets playletmod.PlayletList, findChapters func(context.Context, []int64) (map[int64]*aiContentBackflowChapters, *srvdef.SrvErr), ) ([]aidatacenter.Record, aiContentBackflowStats, int64, *srvdef.SrvErr) { stats := aiContentBackflowStats{DataDate: formatAIContentBackflowDate(targetDate)} ids := make([]int64, 0, len(playlets)) nextID := int64(0) for _, playlet := range playlets { if playlet == nil { continue } ids = append(ids, playlet.Id) if playlet.Id > nextID { nextID = playlet.Id } } chapterByPlayletID, serr := findChapters(ctx, ids) if serr != nil { return nil, stats, nextID, serr } records := make([]aidatacenter.Record, 0, len(playlets)) for _, playlet := range playlets { if playlet == nil { continue } if !shouldReportAIContentBackflowPlaylet(targetDate, playlet) { stats.Skipped++ stats.FuturePublish++ continue } chapters := chapterByPlayletID[playlet.Id] if chapters == nil || chapters.First == nil { stats.Skipped++ stats.MissingChapter++ continue } videoURL := buildAIContentBackflowVideoURL(chapters.First.VideoUrl) if videoURL == "" { stats.Skipped++ stats.MissingVideo++ continue } if chapters.First.MediaContentId == 0 { stats.Skipped++ stats.MissingMediaContent++ continue } records = append(records, makeAIContentBackflowRecord(targetDate, playlet, chapters, videoURL)) stats.Built++ } return records, stats, nextID, nil } // buildAIContentBackflowDailyStats 把短剧模型转换成本地日统计快照。 func buildAIContentBackflowDailyStats( ctx context.Context, targetDate time.Time, playlets playletmod.PlayletList, findChapters func(context.Context, []int64) (map[int64]*aiContentBackflowChapters, *srvdef.SrvErr), ) (playletbackflowstatmod.PlayletBackflowDailyStatList, aiContentBackflowStats, int64, *srvdef.SrvErr) { stats := aiContentBackflowStats{DataDate: formatAIContentBackflowDate(targetDate)} ids := make([]int64, 0, len(playlets)) nextID := int64(0) for _, playlet := range playlets { if playlet == nil { continue } ids = append(ids, playlet.Id) if playlet.Id > nextID { nextID = playlet.Id } } chapterByPlayletID, serr := findChapters(ctx, ids) if serr != nil { return nil, stats, nextID, serr } docs := make(playletbackflowstatmod.PlayletBackflowDailyStatList, 0, len(playlets)) for _, playlet := range playlets { if playlet == nil { continue } if !shouldReportAIContentBackflowPlaylet(targetDate, playlet) { stats.Skipped++ stats.FuturePublish++ continue } chapters := chapterByPlayletID[playlet.Id] if chapters == nil || chapters.First == nil { stats.Skipped++ stats.MissingChapter++ continue } videoURL := buildAIContentBackflowVideoURL(chapters.First.VideoUrl) if videoURL == "" { stats.Skipped++ stats.MissingVideo++ continue } if chapters.First.MediaContentId == 0 { stats.Skipped++ stats.MissingMediaContent++ continue } record := makeAIContentBackflowRecord(targetDate, playlet, chapters, videoURL) docs = append(docs, makeAIContentBackflowDailyStat(targetDate, playlet, chapters.First, record)) stats.Built++ } return docs, stats, nextID, nil } // makeAIContentBackflowRecord 组装一条内容中台数据,缺失指标保持空或 unknown。 func makeAIContentBackflowRecord(targetDate time.Time, playlet *playletmod.Playlet, chapters *aiContentBackflowChapters, videoURL string) aidatacenter.Record { var first, firstPaid *playletchaptermod.PlayletChapter if chapters != nil { first = chapters.First firstPaid = chapters.FirstPaid } views := aiContentBackflowViews(playlet) clicks := aiContentBackflowClicks(first) likes := aiContentBackflowLikes(playlet, first) comments := aiContentBackflowComments(playlet, first) platformURL := "" if first != nil { platformURL = buildAIContentBackflowPlatformURL(playlet.Id, first.Id) } if platformURL == "" { // 内容中台把 platform_url 设为必填;哔哩2是 App-only 产品时没有真实 Web 播放页。 // 使用已验证可访问的视频回源 URL 兜底,避免伪造不存在的播放站点链接。 platformURL = videoURL } record := aidatacenter.Record{ DataDate: formatAIContentBackflowDate(targetDate), CompanyPlatform: strings.TrimSpace(pubdef.CompanyPlatform), PublishedData: formatAIContentBackflowDate(resolveAIContentBackflowPublishedAt(playlet)), ScriptName: strings.TrimSpace(playlet.Title), DramaName: strings.TrimSpace(playlet.Title), DramaNo: aiContentBackflowDramaNo(playlet, first), VideoURL: videoURL, PlatformURL: platformURL, VideoID: aiContentBackflowVideoID(first), PlatformVideoID: aiContentBackflowPlatformVideoID(first), Views: views, Clicks: clicks, Likes: likes, Comments: aidatacenter.Int(comments), Shares: aidatacenter.Int(0), CompletionRateScope: "unknown", } if record.ScriptName == "" { record.ScriptName = record.VideoID } if record.DramaName == "" { record.DramaName = record.VideoID } if clicks > 0 { record.LikeRate = percentString(likes, clicks) } record.PaidRate = aiContentBackflowPaidRate(playlet, firstPaid) return record } // aiContentBackflowVideoID 使用第一集的媒资章节 ID 作为中台视频编号。 func aiContentBackflowVideoID(chapter *playletchaptermod.PlayletChapter) string { if chapter == nil || chapter.MediaContentId == 0 { return "" } return strconv.FormatUint(uint64(chapter.MediaContentId), 10) } // aiContentBackflowPlatformVideoID 使用本项目剧集 ID 作为产品内视频编号。 func aiContentBackflowPlatformVideoID(chapter *playletchaptermod.PlayletChapter) string { if chapter == nil || chapter.Id <= 0 { return "" } return strconv.FormatInt(chapter.Id, 10) } // aiContentBackflowStatVideoID 从日统计快照还原中台视频编号。 func aiContentBackflowStatVideoID(stat *playletbackflowstatmod.PlayletBackflowDailyStat) string { if stat == nil || stat.MediaContentId <= 0 { return "" } return strconv.FormatInt(stat.MediaContentId, 10) } // aiContentBackflowStatPlatformVideoID 从日统计快照还原产品内视频编号。 func aiContentBackflowStatPlatformVideoID(stat *playletbackflowstatmod.PlayletBackflowDailyStat) string { if stat == nil || stat.ChapterId <= 0 { return "" } return strconv.FormatInt(stat.ChapterId, 10) } // aiContentBackflowViews 浏览量使用短剧累计浏览数。 func aiContentBackflowViews(playlet *playletmod.Playlet) int64 { if playlet == nil { return 0 } return nonNegativeInt64(playlet.CountBrowse) } // aiContentBackflowClicks 播放量使用第一集累计播放数。剧集浏览计数在播放时增加。 func aiContentBackflowClicks(chapter *playletchaptermod.PlayletChapter) int64 { if chapter == nil { return 0 } return nonNegativeInt64(chapter.CountBrowse) } // aiContentBackflowLikes 点赞优先用第一集,第一集没有点赞时回退到短剧点赞数。 func aiContentBackflowLikes(playlet *playletmod.Playlet, chapter *playletchaptermod.PlayletChapter) int64 { if chapter != nil && chapter.CountLike > 0 { return chapter.CountLike } if playlet == nil { return 0 } return nonNegativeInt64(playlet.CountLike) } // aiContentBackflowComments 评论优先用第一集,第一集没有评论时回退到短剧评论数。 func aiContentBackflowComments(playlet *playletmod.Playlet, chapter *playletchaptermod.PlayletChapter) int64 { if chapter != nil && chapter.CountComment > 0 { return chapter.CountComment } if playlet == nil { return 0 } return nonNegativeInt64(playlet.CountComment) } // aiContentBackflowPaidRate 付费率 = 第一个付费剧集解锁数 / 短剧浏览数。没有付费剧集时记 0.00%。 func aiContentBackflowPaidRate(playlet *playletmod.Playlet, firstPaid *playletchaptermod.PlayletChapter) string { if firstPaid == nil { return "0.00%" } if playlet == nil || playlet.CountBrowse <= 0 { return "" } return percentString(firstPaid.CountBuy, playlet.CountBrowse) } // makeAIContentBackflowDailyStat 保存一条可观察、可重试的本地统计快照。 func makeAIContentBackflowDailyStat(targetDate time.Time, playlet *playletmod.Playlet, chapter *playletchaptermod.PlayletChapter, record aidatacenter.Record) *playletbackflowstatmod.PlayletBackflowDailyStat { stat := &playletbackflowstatmod.PlayletBackflowDailyStat{ DataDate: record.DataDate, PlayletId: playlet.Id, ChapterId: chapter.Id, CurrentEpisode: chapter.CurrentEpisode, MediaCenterId: int64(playlet.MediaCenterId), MediaContentId: int64(chapter.MediaContentId), Title: strings.TrimSpace(playlet.Title), PublishedData: record.PublishedData, ScriptName: record.ScriptName, DramaName: record.DramaName, DramaNo: record.DramaNo, VideoURL: record.VideoURL, PlatformURL: record.PlatformURL, Views: record.Views, Clicks: record.Clicks, Likes: record.Likes, LikeRate: record.LikeRate, CompletionRate: record.CompletionRate, PaidRate: record.PaidRate, CompletionRateScope: record.CompletionRateScope, ReportStatus: playletbackflowstatmod.ReportStatusPending, } if record.Comments != nil { stat.Comments = *record.Comments } if record.Shares != nil { stat.Shares = *record.Shares } return stat } // makeAIContentBackflowRecordFromStat 还原 SDK records,用于统计表和上报逻辑解耦。 func makeAIContentBackflowRecordFromStat(stat *playletbackflowstatmod.PlayletBackflowDailyStat) aidatacenter.Record { record := aidatacenter.Record{ DataDate: stat.DataDate, CompanyPlatform: strings.TrimSpace(pubdef.CompanyPlatform), PublishedData: stat.PublishedData, ScriptName: stat.ScriptName, DramaName: stat.DramaName, DramaNo: stat.DramaNo, VideoURL: stat.VideoURL, PlatformURL: stat.PlatformURL, VideoID: aiContentBackflowStatVideoID(stat), PlatformVideoID: aiContentBackflowStatPlatformVideoID(stat), Views: stat.Views, Clicks: stat.Clicks, Likes: stat.Likes, LikeRate: stat.LikeRate, CompletionRate: stat.CompletionRate, PaidRate: stat.PaidRate, Comments: aidatacenter.Int(stat.Comments), Shares: aidatacenter.Int(stat.Shares), CompletionRateScope: stat.CompletionRateScope, } return record } // mergeAIContentBackflowStats 合并单页构造统计。 func mergeAIContentBackflowStats(dst *aiContentBackflowStats, src aiContentBackflowStats) { if dst == nil { return } dst.Built += src.Built dst.Stored += src.Stored dst.Sent += src.Sent dst.Batches += src.Batches dst.Reported += src.Reported dst.Failed += src.Failed dst.Skipped += src.Skipped dst.MissingChapter += src.MissingChapter dst.MissingVideo += src.MissingVideo dst.MissingMediaContent += src.MissingMediaContent dst.FuturePublish += src.FuturePublish } // splitAIContentBackflowRecords 按中台批量上限切分 records。 func splitAIContentBackflowRecords(records []aidatacenter.Record, batchSize int64) [][]aidatacenter.Record { if len(records) == 0 { return nil } if batchSize <= 0 { batchSize = defaultAIContentBackflowBatchSize } batchLimit := int(batchSize) if batchLimit <= 0 { batchLimit = int(defaultAIContentBackflowBatchSize) } batches := make([][]aidatacenter.Record, 0, (len(records)+batchLimit-1)/batchLimit) for len(records) > 0 { n := batchLimit if len(records) < n { n = len(records) } batches = append(batches, records[:n]) records = records[n:] } return batches } // shouldReportAIContentBackflowPlaylet 判断短剧在统计日当天是否已经上线。 func shouldReportAIContentBackflowPlaylet(targetDate time.Time, playlet *playletmod.Playlet) bool { publishedAt, ok := resolveAIContentBackflowPublishedAtCandidate(playlet) if !ok { // 没有任何可用时间时保留旧兜底逻辑,交给后续必填校验和服务端校验处理。 return true } return !aiContentBackflowDateOnly(publishedAt).After(aiContentBackflowDateOnly(targetDate)) } // shouldReportAIContentBackflowStat 判断历史快照是否适合在当前数据日上报。 func shouldReportAIContentBackflowStat(stat *playletbackflowstatmod.PlayletBackflowDailyStat) bool { if stat == nil { return false } dataDate, okDataDate := parseAIContentBackflowDate(stat.DataDate) publishedDate, okPublishedDate := parseAIContentBackflowDate(stat.PublishedData) if !okDataDate || !okPublishedDate { // 日期格式异常时不在这里吞掉,继续让 SDK 或内容中台返回明确错误。 return true } return !publishedDate.After(dataDate) } // resolveAIContentBackflowPublishedAt 按上架时间、媒资创建时间、本地创建时间依次兜底。 func resolveAIContentBackflowPublishedAt(playlet *playletmod.Playlet) time.Time { if t, ok := resolveAIContentBackflowPublishedAtCandidate(playlet); ok { return t } return time.Now() } // resolveAIContentBackflowPublishedAtCandidate 返回业务上可确认的短剧上线候选时间。 func resolveAIContentBackflowPublishedAtCandidate(playlet *playletmod.Playlet) (time.Time, bool) { if playlet == nil { return time.Time{}, false } for _, t := range []time.Time{playlet.OnlineAt, playlet.MediaCreatedAt, playlet.CreatedAt} { if !t.IsZero() { return t, true } } return time.Time{}, false } // buildAIContentBackflowVideoURL 按配置的回源地址生成完整 m3u8 链接 func buildAIContentBackflowVideoURL(rawURL string) string { rawURL = strings.TrimSpace(rawURL) if rawURL == "" { return "" } if isAbsoluteHTTPURL(rawURL) { return rawURL } if strings.ContainsAny(rawURL, "\r\n\x00\\") || strings.Contains(strings.ToLower(rawURL), "%2e") || len(rawURL) > 4096 { return "" } parsed, err := url.Parse(rawURL) if err != nil || parsed.IsAbs() || parsed.Host != "" || parsed.User != nil { return "" } name := strings.TrimLeft(parsed.Path, "/") if name == "" || hasAIContentBackflowParentPathSegment(name) { return "" } if ext := path.Ext(name); ext == "" { name += ".m3u8" } else if !strings.EqualFold(ext, ".m3u8") { return "" } result := joinAIContentBackflowURL(resolveAIContentBackflowVideoBaseURL(), name) if result != "" && parsed.RawQuery != "" { result += "?" + parsed.RawQuery } return result } // buildAIContentBackflowPlatformURL 生成产品内播放页 URL。 func buildAIContentBackflowPlatformURL(playletID, chapterID int64) string { return joinAIContentBackflowURL(resolveAIContentBackflowPlatformBaseURL(), fmt.Sprintf("/play?id=%d&chapterId=%d", playletID, chapterID)) } // resolveAIContentBackflowPlatformBaseURL 返回短剧播放页域名。 func resolveAIContentBackflowPlatformBaseURL() string { if baseURL := strings.TrimSpace(aiContentBackflowPlatformBaseURL); baseURL != "" { return strings.TrimRight(baseURL, "/") } return "" } // resolveAIContentBackflowVideoBaseURL 返回数据回流专用视频回源域名。 func resolveAIContentBackflowVideoBaseURL() string { if baseURL := strings.TrimSpace(aiContentBackflowVideoBaseURL); baseURL != "" { return strings.TrimRight(baseURL, "/") } if baseURL := strings.TrimSpace(webg.Conf.DataCenterBackflow.SelfSourceBaseUrl); baseURL != "" { return strings.TrimRight(baseURL, "/") } return "" } // normalizeAIContentBackflowBaseURL 兼容配置里的裸域名和完整 URL。 func normalizeAIContentBackflowBaseURL(host string, listenAddr string) string { host = strings.TrimRight(strings.TrimSpace(host), "/") if host == "" { return "" } if isAbsoluteHTTPURL(host) { return host } if listenAddr = strings.TrimSpace(listenAddr); listenAddr != "" && !strings.Contains(host, ":") { host += ":" + listenAddr } return "https://" + host } // joinAIContentBackflowURL 拼接 baseURL 和相对路径;绝对路径直接返回。 func joinAIContentBackflowURL(baseURL string, path string) string { baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") path = strings.TrimSpace(path) if baseURL == "" || path == "" { return "" } if isAbsoluteHTTPURL(path) { return path } return baseURL + "/" + strings.TrimLeft(path, "/") } // isAbsoluteHTTPURL 判断是否为 http/https 完整 URL。 func isAbsoluteHTTPURL(value string) bool { value = strings.ToLower(strings.TrimSpace(value)) return strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://") } // hasAIContentBackflowParentPathSegment 拒绝路径穿越。 func hasAIContentBackflowParentPathSegment(name string) bool { for _, segment := range strings.Split(name, "/") { if segment == ".." { return true } } return false } // formatAIContentBackflowDate 使用北京时间输出中台要求的 DD/MM/YYYY。 func formatAIContentBackflowDate(t time.Time) string { return t.In(aiContentBackflowLocation()).Format(aidatacenter.DateLayout) } // parseAIContentBackflowDate 解析中台 DD/MM/YYYY 日期为北京时间零点。 func parseAIContentBackflowDate(value string) (time.Time, bool) { t, err := time.ParseInLocation(aidatacenter.DateLayout, strings.TrimSpace(value), aiContentBackflowLocation()) return t, err == nil } // aiContentBackflowDateOnly 只保留北京时间日期,用于比较发布日和统计日。 func aiContentBackflowDateOnly(t time.Time) time.Time { local := t.In(aiContentBackflowLocation()) return time.Date(local.Year(), local.Month(), local.Day(), 0, 0, 0, 0, aiContentBackflowLocation()) } // aiContentBackflowLocation 返回内容中台日期口径使用的北京时间。 func aiContentBackflowLocation() *time.Location { loc, err := time.LoadLocation("Asia/Shanghai") if err != nil { loc = time.Local } return loc } // aiContentBackflowDramaNo 生成中台要求的集数文案。 func aiContentBackflowDramaNo(playlet *playletmod.Playlet, chapter *playletchaptermod.PlayletChapter) string { if chapter != nil && chapter.CurrentEpisode > 0 { return fmt.Sprintf("第%d集", chapter.CurrentEpisode) } if playlet != nil && playlet.TotalEpisode > 0 { return fmt.Sprintf("共%d集", playlet.TotalEpisode) } return "第1集" } // nonNegativeInt64 把业务累计计数转成中台非负 int64。 func nonNegativeInt64(v int64) int64 { if v <= 0 { return 0 } return v } // percentString 生成保留两位小数的百分比字符串。 func percentString(numerator, denominator int64) string { if numerator < 0 { numerator = 0 } if denominator <= 0 { return "" } return fmt.Sprintf("%.2f%%", float64(numerator)*100/float64(denominator)) }