Files
rootandClaude Opus 4.7 79ac89b458 Initial commit
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-11-07 10:47:18 +08:00

1069 lines
38 KiB
Go

package srv_mediaresource
import (
"context"
"fmt"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
"videoweb/pub/pubdef"
"videoweb/pub/common/log"
"videoweb/pub/common/srvdef"
aidatacenter "videoweb/pub/data_center"
"videoweb/pub/models/playletbackflowstatmod"
"videoweb/pub/models/playletchaptermod"
"videoweb/pub/models/playletmod"
"videoweb/web/webg"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
// defaultAIContentBackflowPageSize 每页扫描原创短剧数量,避免一次性加载过多 Mongo 数据。
defaultAIContentBackflowPageSize = int64(200)
// defaultAIContentBackflowBatchSize 每批提交给内容中台的记录数。
defaultAIContentBackflowBatchSize = int64(500)
// maxAIContentBackflowBatchSize 内容中台单批最多允许 1000 条。
maxAIContentBackflowBatchSize = int64(1000)
// defaultAIContentBackflowReportTimeout 单批同步上报的最长等待时间。
defaultAIContentBackflowReportTimeout = 60 * time.Second
)
var (
// aiContentBackflowPlatformBaseURL 覆盖短剧播放页域名;App-only 产品为空时 platform_url 回退到 video_url。
aiContentBackflowPlatformBaseURL = ""
// aiContentBackflowVideoBaseURL 覆盖视频回源域名;空则使用 dataCenterBackflow.selfSourceBaseUrl。
aiContentBackflowVideoBaseURL = ""
// aiContentBackflowPageSize 运行时扫描分页大小,非法值回退默认值。
aiContentBackflowPageSize = int64(200)
// aiContentBackflowBatchSize 运行时上报批量大小,非法值回退默认值。
aiContentBackflowBatchSize = int64(500)
// aiContentBackflowCronSpec 运行时 cron 表达式,空值回退每天 02:30。
aiContentBackflowCronSpec = ""
)
// aiContentBackflowStats 记录一次任务的扫描、构造和跳过数量。
type aiContentBackflowStats struct {
DataDate string // 数据日期,格式 DD/MM/YYYY
Scanned int64 // 扫描到的原创短剧数
Built int64 // 成功构造的回流记录数
Stored int64 // 已写入本地统计表的记录数
Sent int64 // 已发送给内容中台的记录数
Batches int64 // 发送批次数
Reported int64 // 已标记上报成功的记录数
Failed int64 // 已标记上报失败的记录数
Skipped int64 // 因数据缺口跳过的短剧数
MissingChapter int64 // 缺少可用章节的数量
MissingVideo int64 // 缺少视频地址的数量
MissingMediaContent int64 // 第一集缺少媒资章节 ID 的数量
FuturePublish int64 // 上线日期晚于统计日期的数量
}
// aiContentBackflowChapters 一部短剧回流需要的章节:第一集,以及按集数最早的付费集。
type aiContentBackflowChapters struct {
First *playletchaptermod.PlayletChapter
FirstPaid *playletchaptermod.PlayletChapter
}
// aiContentBackflowStore 抽象数据读写和发送函数,方便单测注入假实现。
type aiContentBackflowStore struct {
findPlaylets func(context.Context, bson.M, *options.FindOptions) (playletmod.PlayletList, *srvdef.SrvErr)
findChapters func(context.Context, []int64) (map[int64]*aiContentBackflowChapters, *srvdef.SrvErr)
}
// aiContentBackflowClient 复用 SDK client,避免每次定时任务重复创建后台队列。
var aiContentBackflowClient = struct {
sync.Mutex
client *aidatacenter.Client
apiKey string
endpoint string
}{}
// AIContentBackflowCronSpec 返回原创短剧数据回流的每日调度时间。
func AIContentBackflowCronSpec() string {
if spec := strings.TrimSpace(aiContentBackflowCronSpec); spec != "" {
return spec
}
return "0 30 2 * * ?"
}
// AIContentBackflowTodayStatCronSpec 返回当天统计表小时级刷新时间。
func AIContentBackflowTodayStatCronSpec() string {
return "0 0 * * * ?"
}
// ReportAIContentBackflowYesterday 上报昨日原创短剧数据。
//
// 内容中台日记录按自然日覆盖更新,重复执行同一天不会累加。
func ReportAIContentBackflowYesterday() {
loc, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
loc = time.Local
}
targetDate := time.Now().In(loc).AddDate(0, 0, -1)
stats, serr := reportAIContentBackflow(context.Background(), targetDate)
if serr != nil {
log.Error("AI 内容中台原创短剧数据回流失败", log.Any("err", serr), log.Any("stats", stats))
return
}
if stats != nil {
log.Info("AI 内容中台原创短剧数据回流完成", log.Any("stats", stats))
}
}
// RefreshAIContentBackflowTodayStats 刷新当天原创短剧回流统计,便于上线后观察数据。
func RefreshAIContentBackflowTodayStats() {
loc, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
loc = time.Local
}
targetDate := time.Now().In(loc)
stats, serr := refreshAIContentBackflowStats(context.Background(), targetDate)
if serr != nil {
log.Error("AI 内容中台原创短剧当天统计刷新失败", log.Any("err", serr), log.Any("stats", stats))
return
}
log.Info("AI 内容中台原创短剧当天统计刷新完成", log.Any("stats", stats))
}
// CloseAIContentBackflowClient 关闭复用的内容中台 SDK client。
func CloseAIContentBackflowClient() {
aiContentBackflowClient.Lock()
client := aiContentBackflowClient.client
aiContentBackflowClient.client = nil
aiContentBackflowClient.apiKey = ""
aiContentBackflowClient.endpoint = ""
aiContentBackflowClient.Unlock()
if client != nil {
client.Close()
}
}
// reportAIContentBackflow 从数据回流配置读取 API Key;空 key 表示未启用真实回流。
func reportAIContentBackflow(ctx context.Context, targetDate time.Time) (*aiContentBackflowStats, *srvdef.SrvErr) {
apiKey := strings.TrimSpace(webg.Conf.DataCenterBackflow.Key)
if apiKey == "" {
log.Info("AI 内容中台原创短剧数据回流未启用",
log.Any("dataDate", formatAIContentBackflowDate(targetDate)),
log.Any("reason", "dataCenterBackflow.key empty"),
)
return nil, nil
}
stats, serr := ensureAIContentBackflowStats(ctx, targetDate)
if serr != nil {
return stats, serr
}
client, serr := getAIContentBackflowClient(apiKey)
if serr != nil {
return stats, serr
}
reportStats, serr := reportAIContentBackflowStats(ctx, targetDate, client)
if reportStats != nil {
mergeAIContentBackflowStats(stats, *reportStats)
}
return stats, serr
}
// refreshAIContentBackflowStats 计算指定日期数据并写入本地统计表,不依赖内容中台 API Key。
func refreshAIContentBackflowStats(ctx context.Context, targetDate time.Time) (*aiContentBackflowStats, *srvdef.SrvErr) {
return refreshAIContentBackflowStatsWithStore(ctx, targetDate, aiContentBackflowStore{
findPlaylets: findOriginalPlayletsForBackflow,
findChapters: findFirstPlayletChaptersForBackflow,
})
}
// getAIContentBackflowClient 创建或复用 data_center SDK client。
func getAIContentBackflowClient(apiKey string) (*aidatacenter.Client, *srvdef.SrvErr) {
if !strings.HasPrefix(apiKey, "ld_") {
return nil, srvdef.New(srvdef.Failure, "AI内容中台API Key格式错误", "AI内容中台API Key格式错误")
}
aiContentBackflowClient.Lock()
defer aiContentBackflowClient.Unlock()
if aiContentBackflowClient.client != nil && aiContentBackflowClient.apiKey == apiKey {
return aiContentBackflowClient.client, nil
}
if aiContentBackflowClient.client != nil {
aiContentBackflowClient.client.Close()
}
client := aidatacenter.New(apiKey)
endpoint := client.Endpoint
client.OnError = func(err error) {
log.Error("AI 内容中台原创短剧异步上报失败",
log.Any("endpoint", endpoint),
log.Any("companyPlatform", pubdef.CompanyPlatform),
log.E(err),
)
}
log.Info("AI 内容中台原创短剧数据回流客户端初始化",
log.Any("endpoint", endpoint),
log.Any("companyPlatform", pubdef.CompanyPlatform),
)
aiContentBackflowClient.client = client
aiContentBackflowClient.apiKey = apiKey
aiContentBackflowClient.endpoint = endpoint
return client, nil
}
// refreshAIContentBackflowStatsWithStore 分页扫描原创短剧并写入本地日统计表。
func refreshAIContentBackflowStatsWithStore(ctx context.Context, targetDate time.Time, store aiContentBackflowStore) (*aiContentBackflowStats, *srvdef.SrvErr) {
if serr := validateAIContentBackflowRuntime(); serr != nil {
return nil, serr
}
pageSize := normalizeAIContentBackflowPageSize(aiContentBackflowPageSize)
stats := &aiContentBackflowStats{DataDate: formatAIContentBackflowDate(targetDate)}
log.Info("AI 内容中台原创短剧日统计刷新开始",
log.Any("dataDate", stats.DataDate),
log.Any("companyPlatform", pubdef.CompanyPlatform),
log.Any("pageSize", pageSize),
log.Any("videoBaseURL", resolveAIContentBackflowVideoBaseURL()),
)
for lastID := int64(0); ; {
list, serr := store.findPlaylets(ctx, originalPlayletBackflowFilter(lastID), originalPlayletBackflowFindOptions(pageSize))
if serr != nil {
return stats, serr
}
if len(list) == 0 {
break
}
stats.Scanned += int64(len(list))
docs, pageStats, nextID, serr := buildAIContentBackflowDailyStats(ctx, targetDate, list, store.findChapters)
mergeAIContentBackflowStats(stats, pageStats)
if serr != nil {
return stats, serr
}
if serr := playletbackflowstatmod.UpsertManyPlayletBackflowDailyStats(ctx, docs); serr != nil {
return stats, serr
}
stats.Stored += int64(len(docs))
log.Info("AI 内容中台原创短剧日统计分页刷新完成",
log.Any("dataDate", stats.DataDate),
log.Any("lastID", lastID),
log.Any("nextID", nextID),
log.Any("scanned", len(list)),
log.Any("built", pageStats.Built),
log.Any("stored", len(docs)),
log.Any("skipped", pageStats.Skipped),
log.Any("missingChapter", pageStats.MissingChapter),
log.Any("missingVideo", pageStats.MissingVideo),
log.Any("missingMediaContent", pageStats.MissingMediaContent),
log.Any("futurePublish", pageStats.FuturePublish),
)
if int64(len(list)) < pageSize || nextID <= lastID {
break
}
lastID = nextID
}
return stats, nil
}
// ensureAIContentBackflowStats 确保指定日期已有本地统计,避免每日上报重复扫描。
func ensureAIContentBackflowStats(ctx context.Context, targetDate time.Time) (*aiContentBackflowStats, *srvdef.SrvErr) {
dataDate := formatAIContentBackflowDate(targetDate)
total, serr := playletbackflowstatmod.CountPlayletBackflowDailyStats(ctx, bson.M{
playletbackflowstatmod.FN.PlayletBackflowDailyStat.DataDate: dataDate,
})
if serr != nil {
return nil, serr
}
success, serr := playletbackflowstatmod.CountPlayletBackflowDailyStats(ctx, bson.M{
playletbackflowstatmod.FN.PlayletBackflowDailyStat.DataDate: dataDate,
playletbackflowstatmod.FN.PlayletBackflowDailyStat.ReportStatus: playletbackflowstatmod.ReportStatusSuccess,
})
if serr != nil {
return nil, serr
}
if total > 0 && total == success {
stats := &aiContentBackflowStats{DataDate: dataDate, Stored: total}
log.Info("AI 内容中台原创短剧日统计已全部上报成功",
log.Any("dataDate", stats.DataDate),
log.Any("count", total),
)
return stats, nil
}
log.Info("AI 内容中台原创短剧上报前刷新日统计",
log.Any("dataDate", dataDate),
log.Any("total", total),
log.Any("success", success),
)
return refreshAIContentBackflowStats(ctx, targetDate)
}
// reportAIContentBackflowStats 从本地统计表读取待上报数据,同步发送后标记状态。
func reportAIContentBackflowStats(ctx context.Context, targetDate time.Time, client *aidatacenter.Client) (*aiContentBackflowStats, *srvdef.SrvErr) {
batchSize := normalizeAIContentBackflowBatchSize(aiContentBackflowBatchSize)
stats := &aiContentBackflowStats{DataDate: formatAIContentBackflowDate(targetDate)}
log.Info("AI 内容中台原创短剧统计上报开始",
log.Any("dataDate", stats.DataDate),
log.Any("batchSize", batchSize),
)
for lastID, hasLastID := primitive.NilObjectID, false; ; {
filter := bson.M{
playletbackflowstatmod.FN.PlayletBackflowDailyStat.DataDate: stats.DataDate,
playletbackflowstatmod.FN.PlayletBackflowDailyStat.ReportStatus: bson.M{
"$ne": playletbackflowstatmod.ReportStatusSuccess,
},
}
if hasLastID {
filter[playletbackflowstatmod.FN.PlayletBackflowDailyStat.Id] = bson.M{"$gt": lastID}
}
list, serr := playletbackflowstatmod.FindManyPlayletBackflowDailyStats(ctx, filter, options.Find().
SetSort(bson.D{{Key: playletbackflowstatmod.FN.PlayletBackflowDailyStat.Id, Value: 1}}).
SetLimit(batchSize))
if serr != nil {
return stats, serr
}
if len(list) == 0 {
break
}
records := make([]aidatacenter.Record, 0, len(list))
ids := make([]primitive.ObjectID, 0, len(list))
for _, item := range list {
if item == nil || item.Id.IsZero() {
continue
}
lastID = item.Id
hasLastID = true
if !shouldReportAIContentBackflowStat(item) {
stats.Skipped++
stats.FuturePublish++
log.Warn("AI 内容中台原创短剧统计跳过未来上线快照",
log.Any("dataDate", item.DataDate),
log.Any("playletId", item.PlayletId),
log.Any("publishedData", item.PublishedData),
)
continue
}
if item.MediaContentId <= 0 || item.ChapterId <= 0 {
stats.Skipped++
stats.MissingMediaContent++
log.Warn("AI 内容中台原创短剧统计跳过缺少剧集标识的快照",
log.Any("dataDate", item.DataDate),
log.Any("playletId", item.PlayletId),
log.Any("chapterId", item.ChapterId),
log.Any("mediaContentId", item.MediaContentId),
)
continue
}
records = append(records, makeAIContentBackflowRecordFromStat(item))
ids = append(ids, item.Id)
}
if len(records) == 0 {
if int64(len(list)) < batchSize {
break
}
continue
}
if len(ids) == 0 {
break
}
sendCtx, cancel := context.WithTimeout(ctx, defaultAIContentBackflowReportTimeout)
_, err := client.Send(sendCtx, records...)
cancel()
if err != nil {
reportErr := err.Error()
if serr := playletbackflowstatmod.MarkPlayletBackflowDailyStatsFailed(ctx, ids, reportErr); serr != nil {
return stats, serr
}
stats.Failed += int64(len(ids))
log.Error("AI 内容中台原创短剧统计批次上报失败",
log.Any("dataDate", stats.DataDate),
log.Any("batchSize", len(records)),
log.Any("ids", ids),
log.E(err),
)
return stats, srvdef.New(srvdef.Failure, "AI内容中台原创短剧上报失败", "AI内容中台原创短剧上报失败")
}
reportAt := time.Now()
if serr := playletbackflowstatmod.MarkPlayletBackflowDailyStatsReported(ctx, ids, reportAt); serr != nil {
return stats, serr
}
stats.Sent += int64(len(records))
stats.Reported += int64(len(ids))
stats.Batches++
log.Info("AI 内容中台原创短剧统计批次上报成功",
log.Any("dataDate", stats.DataDate),
log.Any("batchNo", stats.Batches),
log.Any("batchSize", len(records)),
log.Any("sent", stats.Sent),
)
if int64(len(list)) < batchSize {
break
}
}
return stats, nil
}
// validateAIContentBackflowRuntime 校验代码内置口径是否能生成中台必填字段。
func validateAIContentBackflowRuntime() *srvdef.SrvErr {
if strings.TrimSpace(pubdef.CompanyPlatform) == "" {
return srvdef.New(srvdef.Failure, "AI内容中台产品名称未配置", "AI内容中台产品名称未配置")
}
if resolveAIContentBackflowVideoBaseURL() == "" {
return srvdef.New(srvdef.Failure, "AI内容中台视频回源地址未配置", "AI内容中台视频回源地址未配置")
}
return nil
}
// normalizeAIContentBackflowPageSize 修正扫描分页大小。
func normalizeAIContentBackflowPageSize(v int64) int64 {
if v <= 0 {
return defaultAIContentBackflowPageSize
}
return v
}
// normalizeAIContentBackflowBatchSize 修正上报批量大小,并限制在中台上限内。
func normalizeAIContentBackflowBatchSize(v int64) int64 {
if v <= 0 {
return defaultAIContentBackflowBatchSize
}
if v > maxAIContentBackflowBatchSize {
return maxAIContentBackflowBatchSize
}
return v
}
// originalPlayletBackflowFilter 只选择已标记媒资来源且已上架的短剧。
func originalPlayletBackflowFilter(lastID int64) bson.M {
filter := bson.M{
playletmod.FN.Playlet.MediaBusiness: bson.M{"$gt": ""},
playletmod.FN.Playlet.Status: playletmod.StatusUp,
}
if lastID > 0 {
filter[playletmod.FN.Playlet.Id] = bson.M{"$gt": lastID}
}
return filter
}
// originalPlayletBackflowFindOptions 按 ID 游标分页,只读取组装回流记录需要的字段。
func originalPlayletBackflowFindOptions(pageSize int64) *options.FindOptions {
return options.Find().
SetSort(bson.D{{Key: playletmod.FN.Playlet.Id, Value: 1}}).
SetLimit(pageSize).
SetProjection(bson.M{
playletmod.FN.Playlet.Id: 1,
playletmod.FN.Playlet.MediaCenterId: 1,
playletmod.FN.Playlet.Title: 1,
playletmod.FN.Playlet.Status: 1,
playletmod.FN.Playlet.OnlineAt: 1,
playletmod.FN.Playlet.MediaCreatedAt: 1,
playletmod.FN.Playlet.CreatedAt: 1,
playletmod.FN.Playlet.CountBrowse: 1,
playletmod.FN.Playlet.CountLike: 1,
playletmod.FN.Playlet.CountComment: 1,
playletmod.FN.Playlet.TotalEpisode: 1,
})
}
// findOriginalPlayletsForBackflow 查询待回流原创短剧。
func findOriginalPlayletsForBackflow(ctx context.Context, filter bson.M, opts *options.FindOptions) (playletmod.PlayletList, *srvdef.SrvErr) {
return playletmod.FindManyPlaylet(ctx, filter, opts)
}
// findFirstPlayletChaptersForBackflow 取每部短剧的第一集,以及按集数最早的付费集。
func findFirstPlayletChaptersForBackflow(ctx context.Context, playletIDs []int64) (map[int64]*aiContentBackflowChapters, *srvdef.SrvErr) {
if len(playletIDs) == 0 {
return map[int64]*aiContentBackflowChapters{}, nil
}
list, serr := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{
playletchaptermod.FN.PlayletChapter.PlayletId: bson.M{"$in": playletIDs},
}, options.Find().
SetSort(bson.D{
{Key: playletchaptermod.FN.PlayletChapter.PlayletId, Value: 1},
{Key: playletchaptermod.FN.PlayletChapter.CurrentEpisode, Value: 1},
{Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1},
}).
SetProjection(bson.M{
playletchaptermod.FN.PlayletChapter.Id: 1,
playletchaptermod.FN.PlayletChapter.MediaContentId: 1,
playletchaptermod.FN.PlayletChapter.PlayletId: 1,
playletchaptermod.FN.PlayletChapter.CurrentEpisode: 1,
playletchaptermod.FN.PlayletChapter.VideoUrl: 1,
playletchaptermod.FN.PlayletChapter.CountBrowse: 1,
playletchaptermod.FN.PlayletChapter.CountLike: 1,
playletchaptermod.FN.PlayletChapter.CountComment: 1,
playletchaptermod.FN.PlayletChapter.CountBuy: 1,
playletchaptermod.FN.PlayletChapter.PayType: 1,
}))
if serr != nil {
return nil, serr
}
result := make(map[int64]*aiContentBackflowChapters, len(playletIDs))
for _, chapter := range list {
if chapter == nil {
continue
}
item := result[chapter.PlayletId]
if item == nil {
item = &aiContentBackflowChapters{First: chapter}
result[chapter.PlayletId] = item
}
if item.FirstPaid == nil && isPaidPlayletChapter(chapter) {
item.FirstPaid = chapter
}
}
return result, nil
}
// isPaidPlayletChapter 金币或 VIP 剧集需要解锁后才能完整观看。
func isPaidPlayletChapter(chapter *playletchaptermod.PlayletChapter) bool {
if chapter == nil {
return false
}
return chapter.PayType == playletchaptermod.PayTypeCoin || chapter.PayType == playletchaptermod.PayTypeVip
}
// buildAIContentBackflowRecords 把短剧模型转换成内容中台 records。
func buildAIContentBackflowRecords(
ctx context.Context,
targetDate time.Time,
playlets playletmod.PlayletList,
findChapters func(context.Context, []int64) (map[int64]*aiContentBackflowChapters, *srvdef.SrvErr),
) ([]aidatacenter.Record, aiContentBackflowStats, int64, *srvdef.SrvErr) {
stats := aiContentBackflowStats{DataDate: formatAIContentBackflowDate(targetDate)}
ids := make([]int64, 0, len(playlets))
nextID := int64(0)
for _, playlet := range playlets {
if playlet == nil {
continue
}
ids = append(ids, playlet.Id)
if playlet.Id > nextID {
nextID = playlet.Id
}
}
chapterByPlayletID, serr := findChapters(ctx, ids)
if serr != nil {
return nil, stats, nextID, serr
}
records := make([]aidatacenter.Record, 0, len(playlets))
for _, playlet := range playlets {
if playlet == nil {
continue
}
if !shouldReportAIContentBackflowPlaylet(targetDate, playlet) {
stats.Skipped++
stats.FuturePublish++
continue
}
chapters := chapterByPlayletID[playlet.Id]
if chapters == nil || chapters.First == nil {
stats.Skipped++
stats.MissingChapter++
continue
}
videoURL := buildAIContentBackflowVideoURL(chapters.First.VideoUrl)
if videoURL == "" {
stats.Skipped++
stats.MissingVideo++
continue
}
if chapters.First.MediaContentId == 0 {
stats.Skipped++
stats.MissingMediaContent++
continue
}
records = append(records, makeAIContentBackflowRecord(targetDate, playlet, chapters, videoURL))
stats.Built++
}
return records, stats, nextID, nil
}
// buildAIContentBackflowDailyStats 把短剧模型转换成本地日统计快照。
func buildAIContentBackflowDailyStats(
ctx context.Context,
targetDate time.Time,
playlets playletmod.PlayletList,
findChapters func(context.Context, []int64) (map[int64]*aiContentBackflowChapters, *srvdef.SrvErr),
) (playletbackflowstatmod.PlayletBackflowDailyStatList, aiContentBackflowStats, int64, *srvdef.SrvErr) {
stats := aiContentBackflowStats{DataDate: formatAIContentBackflowDate(targetDate)}
ids := make([]int64, 0, len(playlets))
nextID := int64(0)
for _, playlet := range playlets {
if playlet == nil {
continue
}
ids = append(ids, playlet.Id)
if playlet.Id > nextID {
nextID = playlet.Id
}
}
chapterByPlayletID, serr := findChapters(ctx, ids)
if serr != nil {
return nil, stats, nextID, serr
}
docs := make(playletbackflowstatmod.PlayletBackflowDailyStatList, 0, len(playlets))
for _, playlet := range playlets {
if playlet == nil {
continue
}
if !shouldReportAIContentBackflowPlaylet(targetDate, playlet) {
stats.Skipped++
stats.FuturePublish++
continue
}
chapters := chapterByPlayletID[playlet.Id]
if chapters == nil || chapters.First == nil {
stats.Skipped++
stats.MissingChapter++
continue
}
videoURL := buildAIContentBackflowVideoURL(chapters.First.VideoUrl)
if videoURL == "" {
stats.Skipped++
stats.MissingVideo++
continue
}
if chapters.First.MediaContentId == 0 {
stats.Skipped++
stats.MissingMediaContent++
continue
}
record := makeAIContentBackflowRecord(targetDate, playlet, chapters, videoURL)
docs = append(docs, makeAIContentBackflowDailyStat(targetDate, playlet, chapters.First, record))
stats.Built++
}
return docs, stats, nextID, nil
}
// makeAIContentBackflowRecord 组装一条内容中台数据,缺失指标保持空或 unknown。
func makeAIContentBackflowRecord(targetDate time.Time, playlet *playletmod.Playlet, chapters *aiContentBackflowChapters, videoURL string) aidatacenter.Record {
var first, firstPaid *playletchaptermod.PlayletChapter
if chapters != nil {
first = chapters.First
firstPaid = chapters.FirstPaid
}
views := aiContentBackflowViews(playlet)
clicks := aiContentBackflowClicks(first)
likes := aiContentBackflowLikes(playlet, first)
comments := aiContentBackflowComments(playlet, first)
platformURL := ""
if first != nil {
platformURL = buildAIContentBackflowPlatformURL(playlet.Id, first.Id)
}
if platformURL == "" {
// 内容中台把 platform_url 设为必填;哔哩2是 App-only 产品时没有真实 Web 播放页。
// 使用已验证可访问的视频回源 URL 兜底,避免伪造不存在的播放站点链接。
platformURL = videoURL
}
record := aidatacenter.Record{
DataDate: formatAIContentBackflowDate(targetDate),
CompanyPlatform: strings.TrimSpace(pubdef.CompanyPlatform),
PublishedData: formatAIContentBackflowDate(resolveAIContentBackflowPublishedAt(playlet)),
ScriptName: strings.TrimSpace(playlet.Title),
DramaName: strings.TrimSpace(playlet.Title),
DramaNo: aiContentBackflowDramaNo(playlet, first),
VideoURL: videoURL,
PlatformURL: platformURL,
VideoID: aiContentBackflowVideoID(first),
PlatformVideoID: aiContentBackflowPlatformVideoID(first),
Views: views,
Clicks: clicks,
Likes: likes,
Comments: aidatacenter.Int(comments),
Shares: aidatacenter.Int(0),
CompletionRateScope: "unknown",
}
if record.ScriptName == "" {
record.ScriptName = record.VideoID
}
if record.DramaName == "" {
record.DramaName = record.VideoID
}
if clicks > 0 {
record.LikeRate = percentString(likes, clicks)
}
record.PaidRate = aiContentBackflowPaidRate(playlet, firstPaid)
return record
}
// aiContentBackflowVideoID 使用第一集的媒资章节 ID 作为中台视频编号。
func aiContentBackflowVideoID(chapter *playletchaptermod.PlayletChapter) string {
if chapter == nil || chapter.MediaContentId == 0 {
return ""
}
return strconv.FormatUint(uint64(chapter.MediaContentId), 10)
}
// aiContentBackflowPlatformVideoID 使用本项目剧集 ID 作为产品内视频编号。
func aiContentBackflowPlatformVideoID(chapter *playletchaptermod.PlayletChapter) string {
if chapter == nil || chapter.Id <= 0 {
return ""
}
return strconv.FormatInt(chapter.Id, 10)
}
// aiContentBackflowStatVideoID 从日统计快照还原中台视频编号。
func aiContentBackflowStatVideoID(stat *playletbackflowstatmod.PlayletBackflowDailyStat) string {
if stat == nil || stat.MediaContentId <= 0 {
return ""
}
return strconv.FormatInt(stat.MediaContentId, 10)
}
// aiContentBackflowStatPlatformVideoID 从日统计快照还原产品内视频编号。
func aiContentBackflowStatPlatformVideoID(stat *playletbackflowstatmod.PlayletBackflowDailyStat) string {
if stat == nil || stat.ChapterId <= 0 {
return ""
}
return strconv.FormatInt(stat.ChapterId, 10)
}
// aiContentBackflowViews 浏览量使用短剧累计浏览数。
func aiContentBackflowViews(playlet *playletmod.Playlet) int64 {
if playlet == nil {
return 0
}
return nonNegativeInt64(playlet.CountBrowse)
}
// aiContentBackflowClicks 播放量使用第一集累计播放数。剧集浏览计数在播放时增加。
func aiContentBackflowClicks(chapter *playletchaptermod.PlayletChapter) int64 {
if chapter == nil {
return 0
}
return nonNegativeInt64(chapter.CountBrowse)
}
// aiContentBackflowLikes 点赞优先用第一集,第一集没有点赞时回退到短剧点赞数。
func aiContentBackflowLikes(playlet *playletmod.Playlet, chapter *playletchaptermod.PlayletChapter) int64 {
if chapter != nil && chapter.CountLike > 0 {
return chapter.CountLike
}
if playlet == nil {
return 0
}
return nonNegativeInt64(playlet.CountLike)
}
// aiContentBackflowComments 评论优先用第一集,第一集没有评论时回退到短剧评论数。
func aiContentBackflowComments(playlet *playletmod.Playlet, chapter *playletchaptermod.PlayletChapter) int64 {
if chapter != nil && chapter.CountComment > 0 {
return chapter.CountComment
}
if playlet == nil {
return 0
}
return nonNegativeInt64(playlet.CountComment)
}
// aiContentBackflowPaidRate 付费率 = 第一个付费剧集解锁数 / 短剧浏览数。没有付费剧集时记 0.00%。
func aiContentBackflowPaidRate(playlet *playletmod.Playlet, firstPaid *playletchaptermod.PlayletChapter) string {
if firstPaid == nil {
return "0.00%"
}
if playlet == nil || playlet.CountBrowse <= 0 {
return ""
}
return percentString(firstPaid.CountBuy, playlet.CountBrowse)
}
// makeAIContentBackflowDailyStat 保存一条可观察、可重试的本地统计快照。
func makeAIContentBackflowDailyStat(targetDate time.Time, playlet *playletmod.Playlet, chapter *playletchaptermod.PlayletChapter, record aidatacenter.Record) *playletbackflowstatmod.PlayletBackflowDailyStat {
stat := &playletbackflowstatmod.PlayletBackflowDailyStat{
DataDate: record.DataDate,
PlayletId: playlet.Id,
ChapterId: chapter.Id,
CurrentEpisode: chapter.CurrentEpisode,
MediaCenterId: int64(playlet.MediaCenterId),
MediaContentId: int64(chapter.MediaContentId),
Title: strings.TrimSpace(playlet.Title),
PublishedData: record.PublishedData,
ScriptName: record.ScriptName,
DramaName: record.DramaName,
DramaNo: record.DramaNo,
VideoURL: record.VideoURL,
PlatformURL: record.PlatformURL,
Views: record.Views,
Clicks: record.Clicks,
Likes: record.Likes,
LikeRate: record.LikeRate,
CompletionRate: record.CompletionRate,
PaidRate: record.PaidRate,
CompletionRateScope: record.CompletionRateScope,
ReportStatus: playletbackflowstatmod.ReportStatusPending,
}
if record.Comments != nil {
stat.Comments = *record.Comments
}
if record.Shares != nil {
stat.Shares = *record.Shares
}
return stat
}
// makeAIContentBackflowRecordFromStat 还原 SDK records,用于统计表和上报逻辑解耦。
func makeAIContentBackflowRecordFromStat(stat *playletbackflowstatmod.PlayletBackflowDailyStat) aidatacenter.Record {
record := aidatacenter.Record{
DataDate: stat.DataDate,
CompanyPlatform: strings.TrimSpace(pubdef.CompanyPlatform),
PublishedData: stat.PublishedData,
ScriptName: stat.ScriptName,
DramaName: stat.DramaName,
DramaNo: stat.DramaNo,
VideoURL: stat.VideoURL,
PlatformURL: stat.PlatformURL,
VideoID: aiContentBackflowStatVideoID(stat),
PlatformVideoID: aiContentBackflowStatPlatformVideoID(stat),
Views: stat.Views,
Clicks: stat.Clicks,
Likes: stat.Likes,
LikeRate: stat.LikeRate,
CompletionRate: stat.CompletionRate,
PaidRate: stat.PaidRate,
Comments: aidatacenter.Int(stat.Comments),
Shares: aidatacenter.Int(stat.Shares),
CompletionRateScope: stat.CompletionRateScope,
}
return record
}
// mergeAIContentBackflowStats 合并单页构造统计。
func mergeAIContentBackflowStats(dst *aiContentBackflowStats, src aiContentBackflowStats) {
if dst == nil {
return
}
dst.Built += src.Built
dst.Stored += src.Stored
dst.Sent += src.Sent
dst.Batches += src.Batches
dst.Reported += src.Reported
dst.Failed += src.Failed
dst.Skipped += src.Skipped
dst.MissingChapter += src.MissingChapter
dst.MissingVideo += src.MissingVideo
dst.MissingMediaContent += src.MissingMediaContent
dst.FuturePublish += src.FuturePublish
}
// splitAIContentBackflowRecords 按中台批量上限切分 records。
func splitAIContentBackflowRecords(records []aidatacenter.Record, batchSize int64) [][]aidatacenter.Record {
if len(records) == 0 {
return nil
}
if batchSize <= 0 {
batchSize = defaultAIContentBackflowBatchSize
}
batchLimit := int(batchSize)
if batchLimit <= 0 {
batchLimit = int(defaultAIContentBackflowBatchSize)
}
batches := make([][]aidatacenter.Record, 0, (len(records)+batchLimit-1)/batchLimit)
for len(records) > 0 {
n := batchLimit
if len(records) < n {
n = len(records)
}
batches = append(batches, records[:n])
records = records[n:]
}
return batches
}
// shouldReportAIContentBackflowPlaylet 判断短剧在统计日当天是否已经上线。
func shouldReportAIContentBackflowPlaylet(targetDate time.Time, playlet *playletmod.Playlet) bool {
publishedAt, ok := resolveAIContentBackflowPublishedAtCandidate(playlet)
if !ok {
// 没有任何可用时间时保留旧兜底逻辑,交给后续必填校验和服务端校验处理。
return true
}
return !aiContentBackflowDateOnly(publishedAt).After(aiContentBackflowDateOnly(targetDate))
}
// shouldReportAIContentBackflowStat 判断历史快照是否适合在当前数据日上报。
func shouldReportAIContentBackflowStat(stat *playletbackflowstatmod.PlayletBackflowDailyStat) bool {
if stat == nil {
return false
}
dataDate, okDataDate := parseAIContentBackflowDate(stat.DataDate)
publishedDate, okPublishedDate := parseAIContentBackflowDate(stat.PublishedData)
if !okDataDate || !okPublishedDate {
// 日期格式异常时不在这里吞掉,继续让 SDK 或内容中台返回明确错误。
return true
}
return !publishedDate.After(dataDate)
}
// resolveAIContentBackflowPublishedAt 按上架时间、媒资创建时间、本地创建时间依次兜底。
func resolveAIContentBackflowPublishedAt(playlet *playletmod.Playlet) time.Time {
if t, ok := resolveAIContentBackflowPublishedAtCandidate(playlet); ok {
return t
}
return time.Now()
}
// resolveAIContentBackflowPublishedAtCandidate 返回业务上可确认的短剧上线候选时间。
func resolveAIContentBackflowPublishedAtCandidate(playlet *playletmod.Playlet) (time.Time, bool) {
if playlet == nil {
return time.Time{}, false
}
for _, t := range []time.Time{playlet.OnlineAt, playlet.MediaCreatedAt, playlet.CreatedAt} {
if !t.IsZero() {
return t, true
}
}
return time.Time{}, false
}
// buildAIContentBackflowVideoURL 按配置的回源地址生成完整 m3u8 链接
func buildAIContentBackflowVideoURL(rawURL string) string {
rawURL = strings.TrimSpace(rawURL)
if rawURL == "" {
return ""
}
if isAbsoluteHTTPURL(rawURL) {
return rawURL
}
if strings.ContainsAny(rawURL, "\r\n\x00\\") || strings.Contains(strings.ToLower(rawURL), "%2e") || len(rawURL) > 4096 {
return ""
}
parsed, err := url.Parse(rawURL)
if err != nil || parsed.IsAbs() || parsed.Host != "" || parsed.User != nil {
return ""
}
name := strings.TrimLeft(parsed.Path, "/")
if name == "" || hasAIContentBackflowParentPathSegment(name) {
return ""
}
if ext := path.Ext(name); ext == "" {
name += ".m3u8"
} else if !strings.EqualFold(ext, ".m3u8") {
return ""
}
result := joinAIContentBackflowURL(resolveAIContentBackflowVideoBaseURL(), name)
if result != "" && parsed.RawQuery != "" {
result += "?" + parsed.RawQuery
}
return result
}
// buildAIContentBackflowPlatformURL 生成产品内播放页 URL。
func buildAIContentBackflowPlatformURL(playletID, chapterID int64) string {
return joinAIContentBackflowURL(resolveAIContentBackflowPlatformBaseURL(), fmt.Sprintf("/play?id=%d&chapterId=%d", playletID, chapterID))
}
// resolveAIContentBackflowPlatformBaseURL 返回短剧播放页域名。
func resolveAIContentBackflowPlatformBaseURL() string {
if baseURL := strings.TrimSpace(aiContentBackflowPlatformBaseURL); baseURL != "" {
return strings.TrimRight(baseURL, "/")
}
return ""
}
// resolveAIContentBackflowVideoBaseURL 返回数据回流专用视频回源域名。
func resolveAIContentBackflowVideoBaseURL() string {
if baseURL := strings.TrimSpace(aiContentBackflowVideoBaseURL); baseURL != "" {
return strings.TrimRight(baseURL, "/")
}
if baseURL := strings.TrimSpace(webg.Conf.DataCenterBackflow.SelfSourceBaseUrl); baseURL != "" {
return strings.TrimRight(baseURL, "/")
}
return ""
}
// normalizeAIContentBackflowBaseURL 兼容配置里的裸域名和完整 URL。
func normalizeAIContentBackflowBaseURL(host string, listenAddr string) string {
host = strings.TrimRight(strings.TrimSpace(host), "/")
if host == "" {
return ""
}
if isAbsoluteHTTPURL(host) {
return host
}
if listenAddr = strings.TrimSpace(listenAddr); listenAddr != "" && !strings.Contains(host, ":") {
host += ":" + listenAddr
}
return "https://" + host
}
// joinAIContentBackflowURL 拼接 baseURL 和相对路径;绝对路径直接返回。
func joinAIContentBackflowURL(baseURL string, path string) string {
baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
path = strings.TrimSpace(path)
if baseURL == "" || path == "" {
return ""
}
if isAbsoluteHTTPURL(path) {
return path
}
return baseURL + "/" + strings.TrimLeft(path, "/")
}
// isAbsoluteHTTPURL 判断是否为 http/https 完整 URL。
func isAbsoluteHTTPURL(value string) bool {
value = strings.ToLower(strings.TrimSpace(value))
return strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://")
}
// hasAIContentBackflowParentPathSegment 拒绝路径穿越。
func hasAIContentBackflowParentPathSegment(name string) bool {
for _, segment := range strings.Split(name, "/") {
if segment == ".." {
return true
}
}
return false
}
// formatAIContentBackflowDate 使用北京时间输出中台要求的 DD/MM/YYYY。
func formatAIContentBackflowDate(t time.Time) string {
return t.In(aiContentBackflowLocation()).Format(aidatacenter.DateLayout)
}
// parseAIContentBackflowDate 解析中台 DD/MM/YYYY 日期为北京时间零点。
func parseAIContentBackflowDate(value string) (time.Time, bool) {
t, err := time.ParseInLocation(aidatacenter.DateLayout, strings.TrimSpace(value), aiContentBackflowLocation())
return t, err == nil
}
// aiContentBackflowDateOnly 只保留北京时间日期,用于比较发布日和统计日。
func aiContentBackflowDateOnly(t time.Time) time.Time {
local := t.In(aiContentBackflowLocation())
return time.Date(local.Year(), local.Month(), local.Day(), 0, 0, 0, 0, aiContentBackflowLocation())
}
// aiContentBackflowLocation 返回内容中台日期口径使用的北京时间。
func aiContentBackflowLocation() *time.Location {
loc, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
loc = time.Local
}
return loc
}
// aiContentBackflowDramaNo 生成中台要求的集数文案。
func aiContentBackflowDramaNo(playlet *playletmod.Playlet, chapter *playletchaptermod.PlayletChapter) string {
if chapter != nil && chapter.CurrentEpisode > 0 {
return fmt.Sprintf("第%d集", chapter.CurrentEpisode)
}
if playlet != nil && playlet.TotalEpisode > 0 {
return fmt.Sprintf("共%d集", playlet.TotalEpisode)
}
return "第1集"
}
// nonNegativeInt64 把业务累计计数转成中台非负 int64。
func nonNegativeInt64(v int64) int64 {
if v <= 0 {
return 0
}
return v
}
// percentString 生成保留两位小数的百分比字符串。
func percentString(numerator, denominator int64) string {
if numerator < 0 {
numerator = 0
}
if denominator <= 0 {
return ""
}
return fmt.Sprintf("%.2f%%", float64(numerator)*100/float64(denominator))
}