init: 初始化 web_hgdjb_server 项目
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,203 @@
|
||||
package srv_mediaresource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/models/playletmod"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/pub/redis"
|
||||
|
||||
goredis "github.com/go-redis/redis"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const (
|
||||
// PlayletCoverCompressCronSpec 历史空压缩封面扫描频率：每5分钟投递一批
|
||||
PlayletCoverCompressCronSpec = "0 */5 * * * ?"
|
||||
|
||||
playletCoverCompressQueueKeyFmt = "playlet:cover:compress:queue:%d"
|
||||
playletCoverCompressLockKeyFmt = "playlet:cover:compress:lock:%d:%d"
|
||||
playletCoverCompressFailKeyFmt = "playlet:cover:compress:fail:v2:%d:%d"
|
||||
playletCoverCompressLockTTL = 10 * time.Minute
|
||||
playletCoverCompressFailTTL = 10 * time.Minute
|
||||
playletCoverCompressPopWait = 5 * time.Second
|
||||
playletCoverCompressRetryWait = time.Second
|
||||
playletCoverCompressScanSize = int64(200)
|
||||
playletCoverCompressBatchSize = 50
|
||||
)
|
||||
|
||||
func playletCoverCompressQueueKey() string {
|
||||
return fmt.Sprintf(playletCoverCompressQueueKeyFmt, pubdef.AppId)
|
||||
}
|
||||
|
||||
func playletCoverCompressLockKey(playletId int64) string {
|
||||
return fmt.Sprintf(playletCoverCompressLockKeyFmt, pubdef.AppId, playletId)
|
||||
}
|
||||
|
||||
func playletCoverCompressFailKey(playletId int64) string {
|
||||
return fmt.Sprintf(playletCoverCompressFailKeyFmt, pubdef.AppId, playletId)
|
||||
}
|
||||
|
||||
type playletCoverCompressJob struct {
|
||||
PlayletId int64 `json:"playletId"`
|
||||
CreateTime int64 `json:"createTime"`
|
||||
}
|
||||
|
||||
// EnqueuePlayletCoverCompressJobs 扫描压缩封面为空的短剧并投递后台任务
|
||||
func EnqueuePlayletCoverCompressJobs() {
|
||||
ctx := context.Background()
|
||||
enqueued := 0
|
||||
scanned := 0
|
||||
var lastId int64
|
||||
for enqueued < playletCoverCompressBatchSize {
|
||||
filter := emptyCompressedCoverFilter()
|
||||
if lastId > 0 {
|
||||
filter[playletmod.FN.Playlet.Id] = bson.M{"$gt": lastId}
|
||||
}
|
||||
opts := options.Find().
|
||||
SetProjection(bson.D{
|
||||
{Key: playletmod.FN.Playlet.Id, Value: 1},
|
||||
{Key: playletmod.FN.Playlet.Cover, Value: 1},
|
||||
{Key: playletmod.FN.Playlet.CompressedCover, Value: 1},
|
||||
{Key: playletmod.FN.Playlet.Status, Value: 1},
|
||||
}).
|
||||
SetSort(bson.D{{Key: playletmod.FN.Playlet.Id, Value: 1}}).
|
||||
SetLimit(playletCoverCompressScanSize)
|
||||
list, err := playletmod.FindManyPlaylet(ctx, filter, opts)
|
||||
if err != nil {
|
||||
log.Error("扫描待压缩短剧封面失败", log.E(err))
|
||||
return
|
||||
}
|
||||
if len(list) == 0 {
|
||||
break
|
||||
}
|
||||
scanned += len(list)
|
||||
for _, item := range list {
|
||||
if item == nil || item.Id <= 0 {
|
||||
continue
|
||||
}
|
||||
lastId = item.Id
|
||||
if enqueued >= playletCoverCompressBatchSize {
|
||||
break
|
||||
}
|
||||
if has, _ := redis.Exists(playletCoverCompressFailKey(item.Id)); has {
|
||||
continue
|
||||
}
|
||||
if err := pushPlayletCoverCompressJob(ctx, item.Id); err != nil {
|
||||
log.Error("投递短剧封面压缩任务失败", log.Any("playletId", item.Id), log.E(err))
|
||||
continue
|
||||
}
|
||||
enqueued++
|
||||
}
|
||||
if int64(len(list)) < playletCoverCompressScanSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
log.Info("短剧压缩封面任务扫描完成", log.Any("scanCount", scanned), log.Any("enqueued", enqueued))
|
||||
}
|
||||
|
||||
func emptyCompressedCoverFilter() bson.M {
|
||||
return bson.M{
|
||||
playletmod.FN.Playlet.Cover: bson.M{"$nin": []interface{}{"", nil}},
|
||||
"$or": []bson.M{
|
||||
{playletmod.FN.Playlet.CompressedCover: bson.M{"$exists": false}},
|
||||
{playletmod.FN.Playlet.CompressedCover: ""},
|
||||
{playletmod.FN.Playlet.CompressedCover: nil},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func pushPlayletCoverCompressJob(ctx context.Context, playletId int64) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
payload, err := json.Marshal(playletCoverCompressJob{
|
||||
PlayletId: playletId,
|
||||
CreateTime: time.Now().Unix(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return redis.RPush(playletCoverCompressQueueKey(), string(payload))
|
||||
}
|
||||
|
||||
func runPlayletCoverCompressWorker(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
result, err := redis.Client().BLPop(playletCoverCompressPopWait, playletCoverCompressQueueKey()).Result()
|
||||
if err != nil {
|
||||
if err == goredis.Nil {
|
||||
continue
|
||||
}
|
||||
log.Error("消费短剧封面压缩任务失败", log.E(err))
|
||||
time.Sleep(playletCoverCompressRetryWait)
|
||||
continue
|
||||
}
|
||||
if len(result) < 2 {
|
||||
log.Error("短剧封面压缩任务格式异常", log.Any("result", result))
|
||||
continue
|
||||
}
|
||||
handlePlayletCoverCompressPayload(ctx, result[1])
|
||||
}
|
||||
}
|
||||
|
||||
func handlePlayletCoverCompressPayload(ctx context.Context, payload string) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
log.Error("处理短剧封面压缩任务异常", log.Any("err", err), log.Any("payload", payload))
|
||||
}
|
||||
}()
|
||||
var job playletCoverCompressJob
|
||||
if err := json.Unmarshal([]byte(payload), &job); err != nil {
|
||||
log.Error("解析短剧封面压缩任务失败", log.Any("payload", payload), log.E(err))
|
||||
return
|
||||
}
|
||||
if job.PlayletId <= 0 {
|
||||
return
|
||||
}
|
||||
locked, err := redis.Setnx_NewOK(playletCoverCompressLockKey(job.PlayletId), true, playletCoverCompressLockTTL)
|
||||
if err != nil {
|
||||
log.Error("获取短剧封面压缩锁失败", log.Any("playletId", job.PlayletId), log.E(err))
|
||||
return
|
||||
}
|
||||
if !locked {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := redis.Del(playletCoverCompressLockKey(job.PlayletId)); err != nil {
|
||||
log.Error("释放短剧封面压缩锁失败", log.Any("playletId", job.PlayletId), log.E(err))
|
||||
}
|
||||
}()
|
||||
playlet, serr := playletmod.FindOnePlaylet(ctx, bson.M{playletmod.FN.Playlet.Id: job.PlayletId}, options.FindOne().SetProjection(bson.D{
|
||||
{Key: playletmod.FN.Playlet.Id, Value: 1},
|
||||
{Key: playletmod.FN.Playlet.Cover, Value: 1},
|
||||
{Key: playletmod.FN.Playlet.CompressedCover, Value: 1},
|
||||
{Key: playletmod.FN.Playlet.Status, Value: 1},
|
||||
}))
|
||||
if serr != nil || playlet == nil {
|
||||
if serr != nil {
|
||||
log.Warn("查询待压缩短剧失败", log.Any("playletId", job.PlayletId), log.E(serr))
|
||||
}
|
||||
return
|
||||
}
|
||||
if FillPlayletCompressedCover(ctx, playlet) {
|
||||
log.Info("历史短剧压缩封面完成", log.Any("playletId", playlet.Id))
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(playlet.CompressedCover) == "" && strings.TrimSpace(playlet.Cover) != "" {
|
||||
if err := redis.Set(playletCoverCompressFailKey(playlet.Id), "1", playletCoverCompressFailTTL); err != nil {
|
||||
log.Warn("记录短剧封面压缩失败冷却失败", log.Any("playletId", playlet.Id), log.E(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package srv_mediaresource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
"videoweb/pub/common/imgutil"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/netutil"
|
||||
"videoweb/pub/common/playletcache"
|
||||
"videoweb/pub/models/cfg"
|
||||
"videoweb/pub/models/playletmod"
|
||||
"videoweb/pub/storage/proxy"
|
||||
"videoweb/web/service/srv_search"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
// compressPlayletCoverOrEmpty 下载原封面并按 media Resize 规则处理。
|
||||
// 原图未超过 60KB，或原图是 GIF 时，直接返回原封面地址；其它格式超过阈值则压缩为 JPEG 后上传。
|
||||
// 失败返回空字符串，不阻断入库。
|
||||
func compressPlayletCoverOrEmpty(cover string) string {
|
||||
cover = strings.TrimSpace(cover)
|
||||
if cover == "" {
|
||||
return ""
|
||||
}
|
||||
pathStr, err := uploadCompressedPlayletCover(cover)
|
||||
if err != nil {
|
||||
log.Warn("压缩短剧封面失败", log.Any("cover", cover), log.E(err))
|
||||
return ""
|
||||
}
|
||||
return pathStr
|
||||
}
|
||||
|
||||
// FillPlayletCompressedCover 为指定短剧补写压缩封面，已有值则跳过。
|
||||
func FillPlayletCompressedCover(ctx context.Context, playlet *playletmod.Playlet) bool {
|
||||
if playlet == nil || playlet.Id <= 0 {
|
||||
return false
|
||||
}
|
||||
if strings.TrimSpace(playlet.CompressedCover) != "" {
|
||||
return false
|
||||
}
|
||||
cover := strings.TrimSpace(playlet.Cover)
|
||||
if cover == "" {
|
||||
return false
|
||||
}
|
||||
pathStr := compressPlayletCoverOrEmpty(cover)
|
||||
if pathStr == "" {
|
||||
return false
|
||||
}
|
||||
now := time.Now()
|
||||
if err := playletmod.UpdateOnePlaylet(ctx,
|
||||
bson.M{
|
||||
playletmod.FN.Playlet.Id: playlet.Id,
|
||||
"$or": []bson.M{
|
||||
{playletmod.FN.Playlet.CompressedCover: bson.M{"$exists": false}},
|
||||
{playletmod.FN.Playlet.CompressedCover: ""},
|
||||
{playletmod.FN.Playlet.CompressedCover: nil},
|
||||
},
|
||||
},
|
||||
bson.M{"$set": bson.M{
|
||||
playletmod.FN.Playlet.CompressedCover: pathStr,
|
||||
playletmod.FN.Playlet.UpdatedAt: now,
|
||||
}},
|
||||
); err != nil {
|
||||
log.Warn("回写短剧压缩封面失败", log.Any("playletId", playlet.Id), log.E(err))
|
||||
return false
|
||||
}
|
||||
playlet.CompressedCover = pathStr
|
||||
playletcache.ClearHomeCache()
|
||||
latest, serr := playletmod.FindOnePlaylet(ctx, bson.M{playletmod.FN.Playlet.Id: playlet.Id})
|
||||
if serr == nil && latest != nil && latest.Status == playletmod.StatusUp {
|
||||
if syncErr := srv_search.SyncPlayletStatusToES(ctx, []int64{playlet.Id}, latest.Status); syncErr != nil {
|
||||
log.Warn("压缩封面后同步ES失败", log.Any("playletId", playlet.Id), log.E(syncErr))
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// FillPlayletCompressedHorizontalCover 为指定短剧补写压缩横版封面，已有值则跳过。
|
||||
func FillPlayletCompressedHorizontalCover(ctx context.Context, playlet *playletmod.Playlet) bool {
|
||||
if playlet == nil || playlet.Id <= 0 {
|
||||
return false
|
||||
}
|
||||
if strings.TrimSpace(playlet.CompressedHorizontalCover) != "" {
|
||||
return false
|
||||
}
|
||||
cover := strings.TrimSpace(playlet.HorizontalCover)
|
||||
if cover == "" {
|
||||
return false
|
||||
}
|
||||
pathStr := compressPlayletCoverOrEmpty(cover)
|
||||
if pathStr == "" {
|
||||
return false
|
||||
}
|
||||
now := time.Now()
|
||||
if err := playletmod.UpdateOnePlaylet(ctx,
|
||||
bson.M{
|
||||
playletmod.FN.Playlet.Id: playlet.Id,
|
||||
"$or": []bson.M{
|
||||
{playletmod.FN.Playlet.CompressedHorizontalCover: bson.M{"$exists": false}},
|
||||
{playletmod.FN.Playlet.CompressedHorizontalCover: ""},
|
||||
{playletmod.FN.Playlet.CompressedHorizontalCover: nil},
|
||||
},
|
||||
},
|
||||
bson.M{"$set": bson.M{
|
||||
playletmod.FN.Playlet.CompressedHorizontalCover: pathStr,
|
||||
playletmod.FN.Playlet.UpdatedAt: now,
|
||||
}},
|
||||
); err != nil {
|
||||
log.Warn("回写短剧压缩横版封面失败", log.Any("playletId", playlet.Id), log.E(err))
|
||||
return false
|
||||
}
|
||||
playlet.CompressedHorizontalCover = pathStr
|
||||
playletcache.ClearHomeCache()
|
||||
latest, serr := playletmod.FindOnePlaylet(ctx, bson.M{playletmod.FN.Playlet.Id: playlet.Id})
|
||||
if serr == nil && latest != nil && latest.Status == playletmod.StatusUp {
|
||||
if syncErr := srv_search.SyncPlayletStatusToES(ctx, []int64{playlet.Id}, latest.Status); syncErr != nil {
|
||||
log.Warn("压缩横版封面后同步ES失败", log.Any("playletId", playlet.Id), log.E(syncErr))
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func uploadCompressedPlayletCover(cover string) (string, error) {
|
||||
downloadURL, err := resolvePlayletCoverURL(cover)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
downloadURL, err = encodeCoverDownloadURL(downloadURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
data, err := downloadPlayletCover(downloadURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
data = imgutil.DecryptImage(data)
|
||||
if !imgutil.NeedCompress(data) {
|
||||
return cover, nil
|
||||
}
|
||||
fileName := imgutil.SafeCoverFileName(downloadURL)
|
||||
compressed, outName, err := imgutil.CompressCover(data, fileName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
pathStr, err := proxy.UploadAutopublishImage(outName, compressed)
|
||||
if err != nil {
|
||||
log.Warn("media上传压缩封面失败，回退图床", log.Any("cover", cover), log.E(err))
|
||||
pathStr, err = proxy.UploadImg(outName, compressed, "")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("上传压缩封面失败: %w", err)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(pathStr) == "" {
|
||||
return "", fmt.Errorf("上传压缩封面返回空路径")
|
||||
}
|
||||
return pathStr, nil
|
||||
}
|
||||
|
||||
func resolvePlayletCoverURL(cover string) (string, error) {
|
||||
if strings.HasPrefix(cover, "http://") || strings.HasPrefix(cover, "https://") {
|
||||
return cover, nil
|
||||
}
|
||||
sysCfg, serr := cfg.FindOneSysCfg(context.Background(), bson.M{cfg.FN.SysCfg.Id: 1})
|
||||
if serr != nil {
|
||||
return "", fmt.Errorf("查询图片CDN失败: %s", serr.Error())
|
||||
}
|
||||
imageDomain := ""
|
||||
if sysCfg != nil {
|
||||
imageDomain = strings.TrimSpace(sysCfg.ImgWebCdn)
|
||||
}
|
||||
if imageDomain == "" {
|
||||
return "", fmt.Errorf("图片CDN未配置，无法下载相对路径封面")
|
||||
}
|
||||
return netutil.MergeURL(imageDomain, cover), nil
|
||||
}
|
||||
|
||||
// encodeCoverDownloadURL 把路径里的中文等非 ASCII 段转成百分号编码，避免下载失败或二次拼接异常。
|
||||
func encodeCoverDownloadURL(raw string) (string, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("解析封面地址失败: %w", err)
|
||||
}
|
||||
if u.Scheme == "" || u.Host == "" {
|
||||
return "", fmt.Errorf("封面地址无效")
|
||||
}
|
||||
escaped := u.EscapedPath()
|
||||
if escaped != "" {
|
||||
if dec, err := url.PathUnescape(escaped); err == nil {
|
||||
u.Path = dec
|
||||
u.RawPath = ""
|
||||
}
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func downloadPlayletCover(rawURL string) ([]byte, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("下载封面失败: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; cgdj-playlet-cover/1.0)")
|
||||
req.Header.Set("Accept", "image/*")
|
||||
|
||||
cli := &http.Client{Timeout: 90 * time.Second}
|
||||
resp, err := cli.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("下载封面失败: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("下载封面失败，状态码 %d", resp.StatusCode)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 20<<20))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("下载封面失败: %w", err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("下载封面内容为空")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
package srv_mediaresource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/srvdef"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/pub/redis"
|
||||
|
||||
goredis "github.com/go-redis/redis"
|
||||
)
|
||||
|
||||
const (
|
||||
playletResourceImportQueueKeyFmt = "playlet:resource:import:queue:%d"
|
||||
playletResourceImportLockKeyFmt = "playlet:resource:import:lock:%d:%d"
|
||||
playletResourceJHImportQueueKeyFmt = "playlet:resource:jh:import:queue:%d"
|
||||
playletResourceJHImportLockKeyFmt = "playlet:resource:jh:import:lock:%d:%d"
|
||||
playletResourceImportLockTTL = 10 * time.Minute
|
||||
playletResourceImportPopWait = 5 * time.Second
|
||||
playletResourceImportRetryWait = time.Second
|
||||
)
|
||||
|
||||
type playletResourceImportJob struct {
|
||||
Ids []uint `json:"ids"`
|
||||
OperatorId int64 `json:"operatorId"`
|
||||
OperatorName string `json:"operatorName"`
|
||||
CreateTime int64 `json:"createTime"`
|
||||
}
|
||||
|
||||
var playletResourceImportWorker = struct {
|
||||
mu sync.Mutex
|
||||
cancel context.CancelFunc
|
||||
}{}
|
||||
|
||||
func playletResourceImportQueueKey() string {
|
||||
return fmt.Sprintf(playletResourceImportQueueKeyFmt, pubdef.AppId)
|
||||
}
|
||||
|
||||
func playletResourceImportLockKey(mediaId uint) string {
|
||||
return fmt.Sprintf(playletResourceImportLockKeyFmt, pubdef.AppId, mediaId)
|
||||
}
|
||||
|
||||
func playletResourceJHImportQueueKey() string {
|
||||
return fmt.Sprintf(playletResourceJHImportQueueKeyFmt, pubdef.AppId)
|
||||
}
|
||||
|
||||
func playletResourceJHImportLockKey(mediaId uint) string {
|
||||
return fmt.Sprintf(playletResourceJHImportLockKeyFmt, pubdef.AppId, mediaId)
|
||||
}
|
||||
|
||||
// Start 启动短剧媒资导入消费者
|
||||
func Start() {
|
||||
playletResourceImportWorker.mu.Lock()
|
||||
defer playletResourceImportWorker.mu.Unlock()
|
||||
if playletResourceImportWorker.cancel != nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
playletResourceImportWorker.cancel = cancel
|
||||
go runPlayletResourceImportWorker(ctx)
|
||||
go runPlayletResourceJHImportWorker(ctx)
|
||||
go runPlayletSerialSyncWorker(ctx)
|
||||
go runPlayletCoverCompressWorker(ctx)
|
||||
go func() {
|
||||
time.Sleep(10 * time.Second)
|
||||
EnqueuePlayletCoverCompressJobs()
|
||||
}()
|
||||
log.Info("短剧媒资后台消费者已启动")
|
||||
}
|
||||
|
||||
// Stop 停止短剧媒资导入消费者
|
||||
func Stop() {
|
||||
playletResourceImportWorker.mu.Lock()
|
||||
defer playletResourceImportWorker.mu.Unlock()
|
||||
if playletResourceImportWorker.cancel == nil {
|
||||
return
|
||||
}
|
||||
playletResourceImportWorker.cancel()
|
||||
playletResourceImportWorker.cancel = nil
|
||||
log.Info("短剧媒资后台消费者已停止")
|
||||
}
|
||||
|
||||
// pushPlayletResourceImportJob 投递短剧媒资导入任务
|
||||
func pushPlayletResourceImportJob(ctx context.Context, ids []uint, operatorId int64, operatorName string) *srvdef.SrvErr {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Error("投递短剧媒资导入任务取消", log.E(ctx.Err()))
|
||||
return srvdef.New(srvdef.Failure, "提交同步任务失败", "提交同步任务失败")
|
||||
default:
|
||||
}
|
||||
job := playletResourceImportJob{
|
||||
Ids: ids,
|
||||
OperatorId: operatorId,
|
||||
OperatorName: operatorName,
|
||||
CreateTime: time.Now().Unix(),
|
||||
}
|
||||
payload, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
log.Error("序列化短剧媒资导入任务失败", log.Any("ids", ids), log.E(err))
|
||||
return srvdef.New(srvdef.Failure, "提交同步任务失败", "提交同步任务失败")
|
||||
}
|
||||
if err = redis.RPush(playletResourceImportQueueKey(), string(payload)); err != nil {
|
||||
log.Error("投递短剧媒资导入任务失败", log.Any("ids", ids), log.E(err))
|
||||
return srvdef.New(srvdef.Failure, "提交同步任务失败", "提交同步任务失败")
|
||||
}
|
||||
log.Info("投递短剧媒资导入任务成功", log.Any("ids", ids), log.Any("count", len(ids)))
|
||||
return nil
|
||||
}
|
||||
|
||||
// pushPlayletResourceJHImportJob 投递嘉华短剧媒资导入任务
|
||||
func pushPlayletResourceJHImportJob(ctx context.Context, ids []uint, operatorId int64, operatorName string) *srvdef.SrvErr {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Error("投递嘉华短剧媒资导入任务取消", log.E(ctx.Err()))
|
||||
return srvdef.New(srvdef.Failure, "提交同步任务失败", "提交同步任务失败")
|
||||
default:
|
||||
}
|
||||
job := playletResourceImportJob{
|
||||
Ids: ids,
|
||||
OperatorId: operatorId,
|
||||
OperatorName: operatorName,
|
||||
CreateTime: time.Now().Unix(),
|
||||
}
|
||||
payload, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
log.Error("序列化嘉华短剧媒资导入任务失败", log.Any("ids", ids), log.E(err))
|
||||
return srvdef.New(srvdef.Failure, "提交同步任务失败", "提交同步任务失败")
|
||||
}
|
||||
if err = redis.RPush(playletResourceJHImportQueueKey(), string(payload)); err != nil {
|
||||
log.Error("投递嘉华短剧媒资导入任务失败", log.Any("ids", ids), log.E(err))
|
||||
return srvdef.New(srvdef.Failure, "提交同步任务失败", "提交同步任务失败")
|
||||
}
|
||||
log.Info("投递嘉华短剧媒资导入任务成功", log.Any("ids", ids), log.Any("count", len(ids)))
|
||||
return nil
|
||||
}
|
||||
|
||||
// runPlayletResourceImportWorker 循环消费短剧媒资导入队列
|
||||
func runPlayletResourceImportWorker(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
result, err := redis.Client().BLPop(playletResourceImportPopWait, playletResourceImportQueueKey()).Result()
|
||||
if err != nil {
|
||||
if err == goredis.Nil {
|
||||
continue
|
||||
}
|
||||
log.Error("消费短剧媒资导入任务失败", log.E(err))
|
||||
time.Sleep(playletResourceImportRetryWait)
|
||||
continue
|
||||
}
|
||||
if len(result) < 2 {
|
||||
log.Error("短剧媒资导入任务格式异常", log.Any("result", result))
|
||||
continue
|
||||
}
|
||||
handlePlayletResourceImportPayload(ctx, result[1])
|
||||
}
|
||||
}
|
||||
|
||||
// runPlayletResourceJHImportWorker 循环消费嘉华短剧媒资导入队列
|
||||
func runPlayletResourceJHImportWorker(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
result, err := redis.Client().BLPop(playletResourceImportPopWait, playletResourceJHImportQueueKey()).Result()
|
||||
if err != nil {
|
||||
if err == goredis.Nil {
|
||||
continue
|
||||
}
|
||||
log.Error("消费嘉华短剧媒资导入任务失败", log.E(err))
|
||||
time.Sleep(playletResourceImportRetryWait)
|
||||
continue
|
||||
}
|
||||
if len(result) < 2 {
|
||||
log.Error("嘉华短剧媒资导入任务格式异常", log.Any("result", result))
|
||||
continue
|
||||
}
|
||||
handlePlayletResourceJHImportPayload(ctx, result[1])
|
||||
}
|
||||
}
|
||||
|
||||
// handlePlayletResourceImportPayload 处理单个短剧媒资导入任务
|
||||
func handlePlayletResourceImportPayload(ctx context.Context, payload string) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
log.Error("处理短剧媒资导入任务异常", log.Any("err", err), log.Any("payload", payload))
|
||||
}
|
||||
}()
|
||||
var job playletResourceImportJob
|
||||
if err := json.Unmarshal([]byte(payload), &job); err != nil {
|
||||
log.Error("解析短剧媒资导入任务失败", log.Any("payload", payload), log.E(err))
|
||||
return
|
||||
}
|
||||
ids := dedupePlayletResourceIds(job.Ids)
|
||||
if len(ids) == 0 {
|
||||
log.Error("短剧媒资导入任务没有有效媒资ID", log.Any("payload", payload))
|
||||
return
|
||||
}
|
||||
log.Info("开始处理短剧媒资导入任务", log.Any("ids", ids), log.Any("count", len(ids)), log.Any("createTime", job.CreateTime),
|
||||
log.Any("operatorId", job.OperatorId), log.Any("operatorName", job.OperatorName))
|
||||
resp := importPlayletResourceBatch(ctx, ids, job.OperatorId, job.OperatorName)
|
||||
log.Info("短剧媒资导入任务处理完成",
|
||||
log.Any("ids", ids),
|
||||
log.Any("successCount", len(resp.Success)),
|
||||
log.Any("failedCount", len(resp.Failed)),
|
||||
log.Any("failed", resp.Failed))
|
||||
}
|
||||
|
||||
// lockPlayletResourceImportMedia 锁定单条媒资导入
|
||||
func lockPlayletResourceImportMedia(mediaId uint) (bool, error) {
|
||||
return redis.Setnx_NewOK(playletResourceImportLockKey(mediaId), true, playletResourceImportLockTTL)
|
||||
}
|
||||
|
||||
// unlockPlayletResourceImportMedia 释放单条媒资导入锁
|
||||
func unlockPlayletResourceImportMedia(mediaId uint) {
|
||||
if err := redis.Del(playletResourceImportLockKey(mediaId)); err != nil {
|
||||
log.Error("释放短剧媒资导入锁失败", log.Any("mediaId", mediaId), log.E(err))
|
||||
}
|
||||
}
|
||||
|
||||
// lockPlayletResourceJHImportMedia 锁定单条嘉华媒资导入
|
||||
func lockPlayletResourceJHImportMedia(mediaId uint) (bool, error) {
|
||||
return redis.Setnx_NewOK(playletResourceJHImportLockKey(mediaId), true, playletResourceImportLockTTL)
|
||||
}
|
||||
|
||||
// unlockPlayletResourceJHImportMedia 释放单条嘉华媒资导入锁
|
||||
func unlockPlayletResourceJHImportMedia(mediaId uint) {
|
||||
if err := redis.Del(playletResourceJHImportLockKey(mediaId)); err != nil {
|
||||
log.Error("释放嘉华短剧媒资导入锁失败", log.Any("mediaId", mediaId), log.E(err))
|
||||
}
|
||||
}
|
||||
|
||||
// handlePlayletResourceJHImportPayload 处理单个嘉华短剧媒资导入任务
|
||||
func handlePlayletResourceJHImportPayload(ctx context.Context, payload string) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
log.Error("处理嘉华短剧媒资导入任务异常", log.Any("err", err), log.Any("payload", payload))
|
||||
}
|
||||
}()
|
||||
var job playletResourceImportJob
|
||||
if err := json.Unmarshal([]byte(payload), &job); err != nil {
|
||||
log.Error("解析嘉华短剧媒资导入任务失败", log.Any("payload", payload), log.E(err))
|
||||
return
|
||||
}
|
||||
ids := dedupePlayletResourceIds(job.Ids)
|
||||
if len(ids) == 0 {
|
||||
log.Error("嘉华短剧媒资导入任务没有有效媒资ID", log.Any("payload", payload))
|
||||
return
|
||||
}
|
||||
log.Info("开始处理嘉华短剧媒资导入任务", log.Any("ids", ids), log.Any("count", len(ids)), log.Any("createTime", job.CreateTime),
|
||||
log.Any("operatorId", job.OperatorId), log.Any("operatorName", job.OperatorName))
|
||||
resp := importPlayletJHResourceBatch(ctx, ids, job.OperatorId, job.OperatorName)
|
||||
log.Info("嘉华短剧媒资导入任务处理完成",
|
||||
log.Any("ids", ids),
|
||||
log.Any("successCount", len(resp.Success)),
|
||||
log.Any("failedCount", len(resp.Failed)),
|
||||
log.Any("failed", resp.Failed))
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
package srv_mediaresource
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"videoweb/pub/common"
|
||||
"videoweb/pub/common/httputil"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/web/api/proto"
|
||||
)
|
||||
|
||||
const (
|
||||
// playletMediaListErrMsg 短剧媒资列表接口统一错误文案
|
||||
playletMediaListErrMsg = "短剧媒资列表查询失败"
|
||||
// playletMediaSyncErrMsg 短剧媒资同步接口统一错误文案
|
||||
playletMediaSyncErrMsg = "短剧媒资同步失败"
|
||||
// playletMediaDeletedCode 媒资中心资源已删除业务码
|
||||
playletMediaDeletedCode = 7003
|
||||
)
|
||||
|
||||
// playletMediaCenterService 短剧媒资中心客户端
|
||||
type playletMediaCenterService struct {
|
||||
url string // 媒资中心接口域名
|
||||
proId string // 媒资中心商户号
|
||||
secret string // 媒资中心签名密钥，当前接口按源项目逻辑暂不签名
|
||||
}
|
||||
|
||||
// playletMediaCenterConfig 创建短剧媒资中心客户端所需配置
|
||||
type playletMediaCenterConfig struct {
|
||||
Url string // 媒资中心接口域名
|
||||
ProId string // 媒资中心商户号
|
||||
Secret string // 媒资中心签名密钥
|
||||
}
|
||||
|
||||
// playletMediaCenterMsg 媒资中心统一响应结构
|
||||
type playletMediaCenterMsg struct {
|
||||
Code int `json:"code"` // 媒资中心业务码，0 表示成功
|
||||
Msg string `json:"msg"` // 媒资中心业务消息
|
||||
Data json.RawMessage `json:"data"` // 媒资中心业务数据，校验 code 成功后再解析
|
||||
}
|
||||
|
||||
// playletMediaPagination 媒资中心列表分页信息。
|
||||
type playletMediaPagination struct {
|
||||
IsLastPage bool `json:"is_last_page"` // 是否最后一页
|
||||
Page uint `json:"page"` // 当前页码
|
||||
Size uint `json:"size"` // 每页数量
|
||||
Count uint `json:"count"` // 总数量
|
||||
}
|
||||
|
||||
// playletMediaListReq 媒资中心 /services/media/list 查询参数
|
||||
type playletMediaListReq struct {
|
||||
MediaType int `json:"media_type"` // 媒体类型：1-动漫 2-漫画 4-漫画短剧 5-短剧 6-普通短剧
|
||||
Id *uint `json:"id"` // 媒资中心资源ID
|
||||
VloggerId *uint `json:"vlogger_id"` // UP主ID
|
||||
IsUp *int `json:"is_up"` // 是否UP主资源：0-否 1-是
|
||||
Title *string `json:"title"` // 标题筛选
|
||||
Tag *string `json:"tag"` // 标签筛选
|
||||
SyncType int `json:"sync_type"` // 同步类型：1-全量 2-增量
|
||||
Page uint `json:"page"` // 页码
|
||||
Size uint `json:"size"` // 每页数量
|
||||
}
|
||||
|
||||
// playletMediaListResp 媒资中心列表响应 data 节点
|
||||
type playletMediaListResp struct {
|
||||
Data []*proto.PlayletResourceListItem `json:"data"` // 短剧媒资列表
|
||||
Pagination playletMediaPagination `json:"pagination"` // 分页信息
|
||||
}
|
||||
|
||||
// playletMediaJHListResp 媒资中心嘉华短剧列表响应 data 节点
|
||||
type playletMediaJHListResp struct {
|
||||
Data []*proto.PlayletResourceJHListItem `json:"data"` // 嘉华短剧媒资列表
|
||||
Pagination playletMediaPagination `json:"pagination"` // 分页信息
|
||||
}
|
||||
|
||||
// playletMediaContent 媒资中心短剧章节内容
|
||||
type playletMediaContent struct {
|
||||
ID uint `json:"id"` // 媒资内容ID
|
||||
SaleCoin uint `json:"sale_coin"` // 单集售价
|
||||
VloggerId uint `json:"vlogger_id"` // UP主ID
|
||||
Cover string `json:"cover"` // 封面
|
||||
MediaId uint `json:"video_id"` // 媒资中心资源ID
|
||||
MediaType int `json:"media_type"` // 媒体类型
|
||||
EpisodeNumber int `json:"episode_number"` // 第几集
|
||||
Title string `json:"title"` // 章节名
|
||||
Text string `json:"text"` // 文本内容
|
||||
Urls string `json:"urls"` // 资源地址集
|
||||
Height uint `json:"height"` // 高
|
||||
Width uint `json:"width"` // 宽
|
||||
Ratio float64 `json:"ratio"` // 宽高比
|
||||
MediaSize uint `json:"media_size"` // 资源大小
|
||||
PlayTime uint `json:"play_time"` // 播放时长
|
||||
M3u8Url string `json:"m3u8_url"` // m3u8视频地址
|
||||
AudioUrl string `json:"audio_url"` // 音频地址
|
||||
FsResourceId string `json:"fs_resource_id"` // 文件服务资源ID
|
||||
FsResourceStatus string `json:"fs_resource_status"` // 文件服务资源状态
|
||||
HashId string `json:"hash_id"` // 资源hash
|
||||
DataVersion int8 `json:"data_version"` // 数据版本
|
||||
TerminalCdo string `json:"terminal_cdo"` // 上传者
|
||||
CreateTime int64 `json:"create_time"` // 创建时间
|
||||
UpdateTime int64 `json:"update_time"` // 更新时间
|
||||
}
|
||||
|
||||
// playletMediaSyncReq 媒资中心 /services/media/sync 查询参数
|
||||
type playletMediaSyncReq struct {
|
||||
Id uint `json:"id"` // 媒资中心资源ID
|
||||
SyncTime int64 `json:"sync_time"` // 增量同步时间，0 表示拉全量详情
|
||||
}
|
||||
|
||||
// playletMediaSyncResp 媒资中心同步响应 data 节点
|
||||
type playletMediaSyncResp struct {
|
||||
IsDelete bool `json:"-"` // 是否已从媒资中心删除
|
||||
Media *proto.PlayletResourceListItem `json:"media"` // 短剧主资源
|
||||
MediaContentList []playletMediaContent `json:"media_content_list"` // 短剧章节资源
|
||||
}
|
||||
|
||||
// newPlayletMediaCenterService 创建短剧媒资中心客户端
|
||||
func newPlayletMediaCenterService(c playletMediaCenterConfig) *playletMediaCenterService {
|
||||
return &playletMediaCenterService{
|
||||
url: c.Url,
|
||||
proId: c.ProId,
|
||||
secret: c.Secret,
|
||||
}
|
||||
}
|
||||
|
||||
// mediaListParams 构造媒资中心列表查询参数
|
||||
func (m *playletMediaCenterService) mediaListParams(p *playletMediaListReq) map[string]string {
|
||||
params := map[string]string{
|
||||
"media_type": strconv.Itoa(p.MediaType),
|
||||
"sync_type": strconv.Itoa(p.SyncType),
|
||||
"page": strconv.FormatUint(uint64(p.Page), 10),
|
||||
"size": strconv.FormatUint(uint64(p.Size), 10),
|
||||
"pro_id": m.proId,
|
||||
"sign": m.sign(),
|
||||
}
|
||||
if p.Id != nil {
|
||||
params["id"] = strconv.FormatUint(uint64(*p.Id), 10)
|
||||
}
|
||||
if p.VloggerId != nil {
|
||||
params["vlogger_id"] = strconv.FormatUint(uint64(*p.VloggerId), 10)
|
||||
}
|
||||
if p.IsUp != nil {
|
||||
params["is_up"] = strconv.Itoa(*p.IsUp)
|
||||
}
|
||||
if p.Title != nil {
|
||||
params["title"] = *p.Title
|
||||
}
|
||||
if p.Tag != nil {
|
||||
params["tag"] = *p.Tag
|
||||
}
|
||||
return params
|
||||
}
|
||||
|
||||
// mediaSyncParams 构造媒资中心同步查询参数
|
||||
func (m *playletMediaCenterService) mediaSyncParams(p *playletMediaSyncReq) map[string]string {
|
||||
return map[string]string{
|
||||
"id": strconv.FormatUint(uint64(p.Id), 10),
|
||||
"sync_time": strconv.FormatInt(p.SyncTime, 10),
|
||||
"pro_id": m.proId,
|
||||
"sign": m.sign(),
|
||||
}
|
||||
}
|
||||
|
||||
// mediaList 查询短剧媒资列表
|
||||
func (m *playletMediaCenterService) mediaList(p *playletMediaListReq) (result playletMediaListResp, err error) {
|
||||
var retMsg playletMediaCenterMsg
|
||||
params := m.mediaListParams(p)
|
||||
path := "/services/media/list"
|
||||
log.Info("查询短剧媒资列表", log.Any("params", params))
|
||||
code, err := httputil.Client().GetWithJResp(&retMsg, common.BindUrl(m.url, path), nil, params)
|
||||
if err != nil {
|
||||
log.Error("查询短剧媒资列表请求失败", log.Any("code", code), log.E(err))
|
||||
return result, errors.New(playletMediaListErrMsg)
|
||||
}
|
||||
if code != http.StatusOK {
|
||||
log.Error("查询短剧媒资列表状态码异常", log.Any("code", code))
|
||||
return result, errors.New(playletMediaListErrMsg)
|
||||
}
|
||||
if retMsg.Code != 0 {
|
||||
log.Error("查询短剧媒资列表业务码异常", log.Any("retMsg", retMsg))
|
||||
return result, errors.New(playletMediaListErrMsg)
|
||||
}
|
||||
if err = json.Unmarshal(retMsg.Data, &result); err != nil {
|
||||
log.Error("解析短剧媒资列表失败", log.E(err))
|
||||
return result, errors.New(playletMediaListErrMsg)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// mediaJHList 查询嘉华短剧媒资列表
|
||||
func (m *playletMediaCenterService) mediaJHList(p *playletMediaListReq) (result playletMediaJHListResp, err error) {
|
||||
var retMsg playletMediaCenterMsg
|
||||
params := m.mediaListParams(p)
|
||||
path := "/services/media/list"
|
||||
log.Info("查询嘉华短剧媒资列表", log.Any("params", params))
|
||||
code, err := httputil.Client().GetWithJResp(&retMsg, common.BindUrl(m.url, path), nil, params)
|
||||
if err != nil {
|
||||
log.Error("查询嘉华短剧媒资列表请求失败", log.Any("code", code), log.E(err))
|
||||
return result, errors.New(playletMediaListErrMsg)
|
||||
}
|
||||
if code != http.StatusOK {
|
||||
log.Error("查询嘉华短剧媒资列表状态码异常", log.Any("code", code))
|
||||
return result, errors.New(playletMediaListErrMsg)
|
||||
}
|
||||
if retMsg.Code != 0 {
|
||||
log.Error("查询嘉华短剧媒资列表业务码异常", log.Any("retMsg", retMsg))
|
||||
return result, errors.New(playletMediaListErrMsg)
|
||||
}
|
||||
if err = json.Unmarshal(retMsg.Data, &result); err != nil {
|
||||
log.Error("解析嘉华短剧媒资列表失败", log.E(err))
|
||||
return result, errors.New(playletMediaListErrMsg)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// mediaSync 查询短剧媒资详情和章节内容
|
||||
func (m *playletMediaCenterService) mediaSync(p *playletMediaSyncReq) (result playletMediaSyncResp, err error) {
|
||||
var retMsg playletMediaCenterMsg
|
||||
params := m.mediaSyncParams(p)
|
||||
path := "/services/media/sync"
|
||||
code, err := httputil.Client().GetWithJResp(&retMsg, common.BindUrl(m.url, path), nil, params)
|
||||
if err != nil {
|
||||
log.Error("同步短剧媒资详情请求失败", log.Any("params", params), log.Any("code", code), log.E(err))
|
||||
return result, errors.New(playletMediaSyncErrMsg)
|
||||
}
|
||||
if code != http.StatusOK {
|
||||
log.Error("同步短剧媒资详情状态码异常", log.Any("params", params), log.Any("code", code))
|
||||
return result, errors.New(playletMediaSyncErrMsg)
|
||||
}
|
||||
if retMsg.Code == playletMediaDeletedCode {
|
||||
result.IsDelete = true
|
||||
return result, nil
|
||||
}
|
||||
if retMsg.Code != 0 {
|
||||
log.Error("同步短剧媒资详情业务码异常", log.Any("params", params), log.Any("retMsg", retMsg))
|
||||
return result, errors.New(playletMediaSyncErrMsg)
|
||||
}
|
||||
if err = json.Unmarshal(retMsg.Data, &result); err != nil {
|
||||
log.Error("解析短剧媒资详情失败", log.Any("params", params), log.E(err))
|
||||
return result, errors.New(playletMediaSyncErrMsg)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// sign 生成媒资中心签名
|
||||
func (m *playletMediaCenterService) sign() string {
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
package srv_mediaresource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/srvdef"
|
||||
"videoweb/pub/models/playletmod"
|
||||
"videoweb/web/api/proto"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const (
|
||||
// defaultOriginalPlayletBackfillPageSize 与媒资库列表默认分页一致。
|
||||
defaultOriginalPlayletBackfillPageSize = int64(20)
|
||||
// defaultOriginalPlayletBackfillMaxPages 限制启动回填最多扫描 2000 条媒资。
|
||||
defaultOriginalPlayletBackfillMaxPages = int64(100)
|
||||
// maxOriginalPlayletBackfillMaxPages 防止误配置导致启动任务长时间扫全量。
|
||||
maxOriginalPlayletBackfillMaxPages = int64(1000)
|
||||
)
|
||||
|
||||
// originalPlayletBackfillOnce 保证每个进程只触发一次启动回填。
|
||||
var originalPlayletBackfillOnce sync.Once
|
||||
|
||||
// originalPlayletBackfillOptions 控制 CGDJ 原创短剧回填范围。
|
||||
type originalPlayletBackfillOptions struct {
|
||||
DryRun bool // 只预览不写库
|
||||
PageSize int64 // 媒资库分页大小
|
||||
MaxPages int64 // 最多扫描页数
|
||||
}
|
||||
|
||||
// originalPlayletBackfillItem 记录单部已导入短剧的回填结果。
|
||||
type originalPlayletBackfillItem struct {
|
||||
MediaCenterId int64 // 媒资库主资源 ID
|
||||
PlayletId int64 // 本地短剧 ID
|
||||
Title string // 本地短剧标题
|
||||
MediaTitle string // 媒资库原始标题
|
||||
Source string // 本次识别出的媒资来源
|
||||
Before string // 回填前媒资来源
|
||||
Result string // updated / preview / already_marked / conflict / failed
|
||||
}
|
||||
|
||||
// originalPlayletBackfillResp 汇总一次 CGDJ 回填统计。
|
||||
type originalPlayletBackfillResp struct {
|
||||
DryRun bool
|
||||
PageSize int64
|
||||
MaxPages int64
|
||||
Pages int64
|
||||
SourceTotal int64 // 媒资库返回总数
|
||||
SourceScanned int64 // 已扫描媒资条数
|
||||
SourceMatched int64 // 匹配 CGDJ 前缀的媒资条数
|
||||
Imported int64 // 已导入本地短剧库的条数
|
||||
Matched int64 // 需要写入媒资来源的条数
|
||||
Updated int64 // 实际更新成功条数
|
||||
Skipped int64 // 已有相同来源或并发冲突跳过条数
|
||||
HasMore bool // 达到 MaxPages 后仍可能存在后续数据
|
||||
List []*originalPlayletBackfillItem
|
||||
}
|
||||
|
||||
// originalPlayletBackfillStore 抽象外部依赖，方便单测覆盖媒资库和 Mongo。
|
||||
type originalPlayletBackfillStore struct {
|
||||
mediaList func(*playletMediaListReq) (playletMediaListResp, error)
|
||||
findMany func(context.Context, bson.M, *options.FindOptions) (playletmod.PlayletList, *srvdef.SrvErr)
|
||||
updateOne func(context.Context, bson.M, bson.M) *srvdef.SrvErr
|
||||
audit func(context.Context, *originalPlayletBackfillItem)
|
||||
}
|
||||
|
||||
// StartCGDJOriginalBackfillOnce 在 web 启动后异步执行一次 CGDJ 原创短剧标记回填。
|
||||
func StartCGDJOriginalBackfillOnce(ctx context.Context) {
|
||||
originalPlayletBackfillOnce.Do(func() {
|
||||
go runCGDJOriginalBackfillOnce(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// runCGDJOriginalBackfillOnce 执行启动回填，成功后顺手触发一次昨日数据回流。
|
||||
func runCGDJOriginalBackfillOnce(ctx context.Context) {
|
||||
resp, serr := backfillCGDJOriginalPlaylets(ctx, originalPlayletBackfillOptions{DryRun: false})
|
||||
if serr != nil {
|
||||
log.Error("CGDJ 原创短剧标记启动回填失败", log.Any("err", serr), log.Any("resp", resp))
|
||||
return
|
||||
}
|
||||
log.Info("CGDJ 原创短剧标记启动回填完成",
|
||||
log.Any("pages", resp.Pages),
|
||||
log.Any("sourceTotal", resp.SourceTotal),
|
||||
log.Any("sourceScanned", resp.SourceScanned),
|
||||
log.Any("sourceMatched", resp.SourceMatched),
|
||||
log.Any("imported", resp.Imported),
|
||||
log.Any("matched", resp.Matched),
|
||||
log.Any("updated", resp.Updated),
|
||||
log.Any("skipped", resp.Skipped),
|
||||
log.Any("hasMore", resp.HasMore))
|
||||
ReportAIContentBackflowYesterday()
|
||||
}
|
||||
|
||||
// backfillCGDJOriginalPlaylets 从媒资库 CGDJ 短剧列表回填本地短剧 mediaBusiness。
|
||||
// 判断依据只使用源媒资标题前缀和 mediaCenterId，不依赖本地短剧标题，避免运营改名后漏判。
|
||||
func backfillCGDJOriginalPlaylets(ctx context.Context, opts originalPlayletBackfillOptions) (*originalPlayletBackfillResp, *srvdef.SrvErr) {
|
||||
ms := newPlayletMediaCenterFromConfig()
|
||||
return backfillCGDJOriginalPlayletsWithStore(ctx, opts, originalPlayletBackfillStore{
|
||||
mediaList: ms.mediaList,
|
||||
findMany: func(ctx context.Context, filter bson.M, opts *options.FindOptions) (playletmod.PlayletList, *srvdef.SrvErr) {
|
||||
return playletmod.FindManyPlaylet(ctx, filter, opts)
|
||||
},
|
||||
updateOne: playletmod.UpdateOnePlaylet,
|
||||
audit: func(ctx context.Context, item *originalPlayletBackfillItem) {
|
||||
log.Info("回填短剧媒资业务来源", log.Any("playletId", item.PlayletId), log.Any("mediaCenterId", item.MediaCenterId), log.Any("title", item.Title), log.Any("mediaTitle", item.MediaTitle), log.Any("mediaBusiness", item.Source))
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// backfillCGDJOriginalPlayletsWithStore 按媒资库 CGDJ 搜索结果回填本地短剧媒资来源。
|
||||
func backfillCGDJOriginalPlayletsWithStore(ctx context.Context, opts originalPlayletBackfillOptions, store originalPlayletBackfillStore) (*originalPlayletBackfillResp, *srvdef.SrvErr) {
|
||||
pageSize, maxPages, serr := normalizeOriginalPlayletBackfillOptions(opts)
|
||||
if serr != nil {
|
||||
return nil, serr
|
||||
}
|
||||
resp := &originalPlayletBackfillResp{
|
||||
DryRun: opts.DryRun,
|
||||
PageSize: pageSize,
|
||||
MaxPages: maxPages,
|
||||
List: make([]*originalPlayletBackfillItem, 0),
|
||||
}
|
||||
for _, title := range playletMediaBusinessSearchTitles() {
|
||||
searchTitle := title
|
||||
for page := int64(1); page <= maxPages; page++ {
|
||||
data, err := store.mediaList(&playletMediaListReq{
|
||||
MediaType: playletResourceMediaTypeShort,
|
||||
Title: &searchTitle,
|
||||
SyncType: defaultResourceSyncType,
|
||||
Page: uint(page),
|
||||
Size: uint(pageSize),
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("查询 CGDJ 短剧媒资失败", log.Any("title", searchTitle), log.Any("page", page), log.E(err))
|
||||
return resp, srvdef.New(srvdef.Failure, "查询 CGDJ 短剧媒资失败", "查询 CGDJ 短剧媒资失败")
|
||||
}
|
||||
resp.Pages++
|
||||
if page == 1 {
|
||||
resp.SourceTotal += int64(data.Pagination.Count)
|
||||
}
|
||||
if len(data.Data) == 0 {
|
||||
break
|
||||
}
|
||||
if serr = backfillOriginalPlayletPage(ctx, data.Data, resp, store); serr != nil {
|
||||
return resp, serr
|
||||
}
|
||||
if data.Pagination.IsLastPage || int64(len(data.Data)) < pageSize {
|
||||
break
|
||||
}
|
||||
if page == maxPages {
|
||||
resp.HasMore = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// normalizeOriginalPlayletBackfillOptions 修正分页参数并限制启动扫描范围。
|
||||
func normalizeOriginalPlayletBackfillOptions(opts originalPlayletBackfillOptions) (int64, int64, *srvdef.SrvErr) {
|
||||
pageSize := opts.PageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = defaultOriginalPlayletBackfillPageSize
|
||||
}
|
||||
maxPages := opts.MaxPages
|
||||
if maxPages <= 0 {
|
||||
maxPages = defaultOriginalPlayletBackfillMaxPages
|
||||
}
|
||||
if pageSize > int64(maxResourcePageSize) {
|
||||
return 0, 0, srvdef.New(srvdef.ErrParamError, "pageSize最多100", "pageSize最多100")
|
||||
}
|
||||
if maxPages > maxOriginalPlayletBackfillMaxPages {
|
||||
return 0, 0, srvdef.New(srvdef.ErrParamError, "maxPages最多1000", "maxPages最多1000")
|
||||
}
|
||||
return pageSize, maxPages, nil
|
||||
}
|
||||
|
||||
// backfillOriginalPlayletPage 只更新已导入本地短剧库的 CGDJ 媒资。
|
||||
func backfillOriginalPlayletPage(ctx context.Context, mediaList []*proto.PlayletResourceListItem, resp *originalPlayletBackfillResp, store originalPlayletBackfillStore) *srvdef.SrvErr {
|
||||
mediaTitleByID := make(map[uint]string, len(mediaList))
|
||||
mediaBusinessByID := make(map[uint]string, len(mediaList))
|
||||
mediaIDs := make([]uint, 0, len(mediaList))
|
||||
for _, media := range mediaList {
|
||||
resp.SourceScanned++
|
||||
business := playletMediaBusiness(media)
|
||||
if business == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := mediaTitleByID[media.ID]; exists {
|
||||
continue
|
||||
}
|
||||
resp.SourceMatched++
|
||||
mediaTitleByID[media.ID] = media.Title
|
||||
mediaBusinessByID[media.ID] = business
|
||||
mediaIDs = append(mediaIDs, media.ID)
|
||||
}
|
||||
if len(mediaIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
playlets, serr := store.findMany(ctx, bson.M{
|
||||
playletmod.FN.Playlet.MediaCenterId: bson.M{"$in": mediaIDs},
|
||||
playletmod.FN.Playlet.Status: playletmod.StatusUp,
|
||||
}, options.Find().SetProjection(bson.M{
|
||||
playletmod.FN.Playlet.Id: 1,
|
||||
playletmod.FN.Playlet.MediaCenterId: 1,
|
||||
playletmod.FN.Playlet.Title: 1,
|
||||
playletmod.FN.Playlet.MediaBusiness: 1,
|
||||
}))
|
||||
if serr != nil {
|
||||
log.Error("查询已导入 CGDJ 短剧失败", log.Any("mediaIds", mediaIDs), log.Any("err", serr))
|
||||
return serr
|
||||
}
|
||||
for _, playlet := range playlets {
|
||||
if playlet == nil || playlet.MediaCenterId == 0 {
|
||||
continue
|
||||
}
|
||||
business := mediaBusinessByID[playlet.MediaCenterId]
|
||||
if business == "" {
|
||||
continue
|
||||
}
|
||||
resp.Imported++
|
||||
item := &originalPlayletBackfillItem{
|
||||
MediaCenterId: int64(playlet.MediaCenterId),
|
||||
PlayletId: playlet.Id,
|
||||
Title: playlet.Title,
|
||||
MediaTitle: mediaTitleByID[playlet.MediaCenterId],
|
||||
Source: business,
|
||||
Before: playlet.MediaBusiness,
|
||||
}
|
||||
if playlet.MediaBusiness == business {
|
||||
resp.Skipped++
|
||||
item.Result = "already_marked"
|
||||
resp.List = append(resp.List, item)
|
||||
continue
|
||||
}
|
||||
resp.Matched++
|
||||
if resp.DryRun {
|
||||
item.Result = "preview"
|
||||
resp.List = append(resp.List, item)
|
||||
continue
|
||||
}
|
||||
serr = store.updateOne(ctx, bson.M{
|
||||
playletmod.FN.Playlet.Id: playlet.Id,
|
||||
playletmod.FN.Playlet.MediaCenterId: playlet.MediaCenterId,
|
||||
playletmod.FN.Playlet.Status: playletmod.StatusUp,
|
||||
playletmod.FN.Playlet.MediaBusiness: bson.M{"$ne": business},
|
||||
}, bson.M{"$set": bson.M{
|
||||
playletmod.FN.Playlet.MediaBusiness: business,
|
||||
}})
|
||||
if serr != nil {
|
||||
if serr.Code == srvdef.ErrNoData {
|
||||
resp.Skipped++
|
||||
item.Result = "conflict"
|
||||
resp.List = append(resp.List, item)
|
||||
continue
|
||||
}
|
||||
item.Result = "failed"
|
||||
resp.List = append(resp.List, item)
|
||||
return serr
|
||||
}
|
||||
resp.Updated++
|
||||
item.Result = "updated"
|
||||
resp.List = append(resp.List, item)
|
||||
if store.audit != nil {
|
||||
store.audit(ctx, item)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,528 @@
|
||||
package srv_mediaresource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/srvdef"
|
||||
"videoweb/pub/models/playletchaptermod"
|
||||
"videoweb/pub/models/playletmod"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/pub/redis"
|
||||
"videoweb/web/api/proto"
|
||||
|
||||
goredis "github.com/go-redis/redis"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const (
|
||||
// PlayletSerialSyncCronSpec 连载中短剧同步扫描频率：每1小时扫描一次，扫描阶段只投递 Redis 任务
|
||||
PlayletSerialSyncCronSpec = "0 0 * * * ?"
|
||||
|
||||
// playletSerialSyncQueueKeyFmt 解耦定时扫描和三方同步调用，避免 cron 扫描过程直接阻塞在外部接口上
|
||||
playletSerialSyncQueueKeyFmt = "playlet:serial:sync:queue:%d"
|
||||
playletSerialSyncLockKeyFmt = "playlet:serial:sync:lock:%d:%d"
|
||||
playletSerialSyncJHLockKeyFmt = "playlet:serial:sync:jh:lock:%d:%d"
|
||||
playletSerialSyncLockTTL = 55 * time.Minute
|
||||
playletSerialSyncPopWait = 5 * time.Second
|
||||
playletSerialSyncRetryWait = time.Second
|
||||
playletSerialSyncBatchSize = int64(100)
|
||||
playletManualSyncMaxCount = 100
|
||||
)
|
||||
|
||||
func playletSerialSyncQueueKey() string {
|
||||
return fmt.Sprintf(playletSerialSyncQueueKeyFmt, pubdef.AppId)
|
||||
}
|
||||
|
||||
func playletSerialSyncLockKey(mediaId uint) string {
|
||||
return fmt.Sprintf(playletSerialSyncLockKeyFmt, pubdef.AppId, mediaId)
|
||||
}
|
||||
|
||||
func playletSerialSyncJHLockKey(mediaId uint) string {
|
||||
return fmt.Sprintf(playletSerialSyncJHLockKeyFmt, pubdef.AppId, mediaId)
|
||||
}
|
||||
|
||||
type playletSerialSyncJob struct {
|
||||
PlayletId int64 `json:"playletId"`
|
||||
MediaCenterId uint `json:"mediaCenterId"`
|
||||
MediaSource string `json:"mediaSource"`
|
||||
MediaSyncTime int64 `json:"mediaSyncTime"`
|
||||
CreateTime int64 `json:"createTime"`
|
||||
}
|
||||
|
||||
// EnqueueSerialPlayletSyncJobs 扫描连载中短剧并投递 Redis 同步任务
|
||||
//
|
||||
// 这里不直接请求三方媒资中心：扫描只负责发现任务，实际同步由后台消费者逐条处理
|
||||
// 可以降低定时任务执行时间，也方便 Redis 队列在三方接口短暂异常时继续缓冲任务
|
||||
func EnqueueSerialPlayletSyncJobs() {
|
||||
ctx := context.Background()
|
||||
filter := bson.M{
|
||||
playletmod.FN.Playlet.MediaCenterId: bson.M{"$gt": 0},
|
||||
playletmod.FN.Playlet.UpdateStatus: 1,
|
||||
// 原创短剧由后台维护剧集，自动同步只处理媒资导入剧
|
||||
playletmod.FN.Playlet.IsOriginal: bson.M{"$ne": true},
|
||||
}
|
||||
opts := options.Find().
|
||||
SetSort(bson.D{{Key: playletmod.FN.Playlet.UpdatedAt, Value: 1}, {Key: playletmod.FN.Playlet.Id, Value: 1}}).
|
||||
SetLimit(playletSerialSyncBatchSize)
|
||||
list, err := playletmod.FindManyPlaylet(ctx, filter, opts)
|
||||
if err != nil {
|
||||
log.Error("扫描连载中短剧失败", log.Any("err", err))
|
||||
return
|
||||
}
|
||||
enqueued := 0
|
||||
for _, item := range list {
|
||||
if item == nil || item.Id == 0 || item.MediaCenterId == 0 || item.IsOriginal {
|
||||
continue
|
||||
}
|
||||
if err := pushPlayletSerialSyncJob(ctx, item); err != nil {
|
||||
log.Error("投递连载短剧同步任务失败", log.Any("playletId", item.Id), log.Any("mediaCenterId", item.MediaCenterId), log.E(err))
|
||||
continue
|
||||
}
|
||||
enqueued++
|
||||
}
|
||||
log.Info("连载中短剧同步任务扫描完成", log.Any("scanCount", len(list)), log.Any("enqueued", enqueued))
|
||||
}
|
||||
|
||||
// pushPlayletSerialSyncJob 将单部连载短剧投递到 Redis 队列
|
||||
func pushPlayletSerialSyncJob(ctx context.Context, playlet *playletmod.Playlet) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
job := playletSerialSyncJob{
|
||||
PlayletId: playlet.Id,
|
||||
MediaCenterId: playlet.MediaCenterId,
|
||||
MediaSource: playlet.MediaSource,
|
||||
MediaSyncTime: playlet.MediaSyncTime,
|
||||
CreateTime: time.Now().Unix(),
|
||||
}
|
||||
payload, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return redis.RPush(playletSerialSyncQueueKey(), string(payload))
|
||||
}
|
||||
|
||||
// runPlayletSerialSyncWorker 常驻消费连载短剧同步队列
|
||||
func runPlayletSerialSyncWorker(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
result, err := redis.Client().BLPop(playletSerialSyncPopWait, playletSerialSyncQueueKey()).Result()
|
||||
if err != nil {
|
||||
if err == goredis.Nil {
|
||||
continue
|
||||
}
|
||||
log.Error("消费连载短剧同步任务失败", log.E(err))
|
||||
time.Sleep(playletSerialSyncRetryWait)
|
||||
continue
|
||||
}
|
||||
if len(result) < 2 {
|
||||
log.Error("连载短剧同步任务格式异常", log.Any("result", result))
|
||||
continue
|
||||
}
|
||||
handlePlayletSerialSyncPayload(ctx, result[1])
|
||||
}
|
||||
}
|
||||
|
||||
// handlePlayletSerialSyncPayload 解析单个 Redis 任务并执行同步，panic 会被捕获避免 worker 退出
|
||||
func handlePlayletSerialSyncPayload(ctx context.Context, payload string) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
log.Error("处理连载短剧同步任务异常", log.Any("err", err), log.Any("payload", payload))
|
||||
}
|
||||
}()
|
||||
var job playletSerialSyncJob
|
||||
if err := json.Unmarshal([]byte(payload), &job); err != nil {
|
||||
log.Error("解析连载短剧同步任务失败", log.Any("payload", payload), log.E(err))
|
||||
return
|
||||
}
|
||||
if job.PlayletId <= 0 || job.MediaCenterId == 0 {
|
||||
log.Error("连载短剧同步任务参数异常", log.Any("job", job))
|
||||
return
|
||||
}
|
||||
if _, reason := syncSerialPlaylet(ctx, job, false, false); reason != "" {
|
||||
log.Error("连载短剧同步任务失败", log.Any("job", job), log.Any("reason", reason))
|
||||
return
|
||||
}
|
||||
log.Info("连载短剧同步任务完成", log.Any("job", job))
|
||||
}
|
||||
|
||||
// SyncPlayletChapters 手动同步指定短剧的媒资剧集，支持单/批量，允许已完结短剧补集或修正状态。
|
||||
func SyncPlayletChapters(ctx context.Context, req *proto.PlayletChapterSyncReq) (*proto.PlayletChapterSyncResp, *srvdef.SrvErr) {
|
||||
ids := normalizePlayletChapterSyncIds(req)
|
||||
if len(ids) == 0 {
|
||||
return nil, srvdef.New(srvdef.ErrParamError, "短剧ID不能为空", "")
|
||||
}
|
||||
if len(ids) > playletManualSyncMaxCount {
|
||||
return nil, srvdef.New(srvdef.ErrParamError, "单次最多同步100部短剧", "")
|
||||
}
|
||||
resp := &proto.PlayletChapterSyncResp{
|
||||
Total: len(ids),
|
||||
List: make([]*proto.PlayletChapterSyncResult, 0, len(ids)),
|
||||
}
|
||||
for _, playletId := range ids {
|
||||
item := syncOnePlayletChapters(ctx, playletId)
|
||||
if item.Success {
|
||||
resp.SuccessCount++
|
||||
} else {
|
||||
resp.FailedCount++
|
||||
}
|
||||
resp.List = append(resp.List, item)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// normalizePlayletChapterSyncIds 过滤重复或无效短剧ID。
|
||||
func normalizePlayletChapterSyncIds(req *proto.PlayletChapterSyncReq) []int64 {
|
||||
if req == nil {
|
||||
return nil
|
||||
}
|
||||
result := make([]int64, 0, len(req.PlayletIds))
|
||||
seen := make(map[int64]struct{}, len(req.PlayletIds))
|
||||
for _, id := range req.PlayletIds {
|
||||
if id <= 0 {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
result = append(result, id)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// syncOnePlayletChapters 同步单部短剧的媒资剧集，单部失败不影响批量里其他短剧。
|
||||
func syncOnePlayletChapters(ctx context.Context, playletId int64) *proto.PlayletChapterSyncResult {
|
||||
result := &proto.PlayletChapterSyncResult{
|
||||
PlayletId: playletId,
|
||||
}
|
||||
playlet, err := playletmod.FindOnePlaylet(ctx, bson.M{
|
||||
playletmod.FN.Playlet.Id: playletId,
|
||||
}, options.FindOne().SetProjection(bson.D{
|
||||
{Key: playletmod.FN.Playlet.Id, Value: 1},
|
||||
{Key: playletmod.FN.Playlet.MediaCenterId, Value: 1},
|
||||
{Key: playletmod.FN.Playlet.MediaSource, Value: 1},
|
||||
{Key: playletmod.FN.Playlet.MediaSyncTime, Value: 1},
|
||||
}))
|
||||
if err != nil {
|
||||
result.Reason = playletChapterSyncErrReason(err)
|
||||
return result
|
||||
}
|
||||
if playlet.MediaCenterId == 0 {
|
||||
result.Reason = "非媒资导入短剧不能同步剧集"
|
||||
return result
|
||||
}
|
||||
newChapterCount, reason := syncSerialPlaylet(ctx, playletSerialSyncJob{
|
||||
PlayletId: playlet.Id,
|
||||
MediaCenterId: playlet.MediaCenterId,
|
||||
MediaSource: playlet.MediaSource,
|
||||
MediaSyncTime: playlet.MediaSyncTime,
|
||||
CreateTime: time.Now().Unix(),
|
||||
}, true, true)
|
||||
if reason != "" {
|
||||
result.Reason = reason
|
||||
return result
|
||||
}
|
||||
result.Success = true
|
||||
result.NewChapterCount = newChapterCount
|
||||
return result
|
||||
}
|
||||
|
||||
// playletChapterSyncErrReason 转换单部短剧同步失败原因，避免批量结果里 reason 为空。
|
||||
func playletChapterSyncErrReason(err *srvdef.SrvErr) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
if err.Code == srvdef.ErrNoData {
|
||||
return "短剧不存在"
|
||||
}
|
||||
if reason := err.GetTip(); reason != "" {
|
||||
return reason
|
||||
}
|
||||
if reason := err.GetMsg(); reason != "" {
|
||||
return reason
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
// syncSerialPlaylet 更新主表集数/状态，只追加新增章节，不覆盖已有章节。
|
||||
// 手动同步使用 forceFullSync=true 全量拉取，便于恢复本地误删的旧章节；定时任务继续使用增量同步。
|
||||
func syncSerialPlaylet(ctx context.Context, job playletSerialSyncJob, allowCompleted bool, forceFullSync bool) (int, string) {
|
||||
lockSource := job.MediaSource
|
||||
locked, err := lockPlayletSerialSyncBySource(lockSource, job.MediaCenterId)
|
||||
if err != nil {
|
||||
log.Error("获取连载短剧同步锁失败", log.Any("job", job), log.E(err))
|
||||
return 0, "获取同步锁失败"
|
||||
}
|
||||
if !locked {
|
||||
return 0, "该媒资正在同步"
|
||||
}
|
||||
defer unlockPlayletSerialSyncBySource(lockSource, job.MediaCenterId)
|
||||
|
||||
playlet, serr := playletmod.FindOnePlaylet(ctx, bson.M{
|
||||
playletmod.FN.Playlet.Id: job.PlayletId,
|
||||
playletmod.FN.Playlet.MediaCenterId: job.MediaCenterId,
|
||||
})
|
||||
if serr != nil {
|
||||
if serr.Code == srvdef.ErrNoData {
|
||||
return 0, "短剧不存在"
|
||||
}
|
||||
log.Error("查询连载短剧失败", log.Any("job", job), log.Any("err", serr))
|
||||
return 0, "查询短剧失败"
|
||||
}
|
||||
if !allowCompleted && playlet.IsOriginal {
|
||||
return 0, ""
|
||||
}
|
||||
if !allowCompleted && playlet.UpdateStatus != 1 {
|
||||
return 0, ""
|
||||
}
|
||||
|
||||
mediaSource := playlet.MediaSource
|
||||
if mediaSource == "" {
|
||||
mediaSource = job.MediaSource
|
||||
}
|
||||
ms := newPlayletMediaCenterBySource(mediaSource)
|
||||
syncTime := playlet.MediaSyncTime
|
||||
if forceFullSync {
|
||||
syncTime = 0
|
||||
}
|
||||
result, err := ms.mediaSync(&playletMediaSyncReq{
|
||||
Id: playlet.MediaCenterId,
|
||||
SyncTime: syncTime,
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("同步连载短剧媒资失败", log.Any("playletId", playlet.Id), log.Any("mediaCenterId", playlet.MediaCenterId), log.E(err))
|
||||
return 0, "同步媒资失败"
|
||||
}
|
||||
if result.IsDelete {
|
||||
return 0, "媒资不存在或已删除"
|
||||
}
|
||||
if result.Media == nil {
|
||||
return 0, "媒资主资源为空"
|
||||
}
|
||||
|
||||
newSyncTime := maxPlayletMediaContentUpdateTime(playlet.MediaSyncTime, result.MediaContentList)
|
||||
if reason := updateSerialPlayletInfo(ctx, playlet, result.Media, newSyncTime); reason != "" {
|
||||
return 0, reason
|
||||
}
|
||||
if len(result.MediaContentList) == 0 {
|
||||
return 0, ""
|
||||
}
|
||||
chapters, reason := buildNewSerialPlayletChapters(ctx, playlet.Id, mediaSource, result.MediaContentList)
|
||||
if reason != "" {
|
||||
return 0, reason
|
||||
}
|
||||
if len(chapters) == 0 {
|
||||
return 0, ""
|
||||
}
|
||||
// 上架且连载中时，新增剧集默认支付方式取剧级设置。
|
||||
applyPlayletDefaultPayToChapters(playlet, chapters)
|
||||
if serr = playletchaptermod.InsertManyPlayletChapter(ctx, chapters); serr != nil {
|
||||
log.Error("连载短剧新增章节入库失败", log.Any("playletId", playlet.Id), log.Any("mediaCenterId", playlet.MediaCenterId), log.Any("count", len(chapters)), log.Any("err", serr))
|
||||
return 0, "新增章节入库失败"
|
||||
}
|
||||
log.Info("连载短剧新增章节成功", log.Any("playletId", playlet.Id), log.Any("mediaCenterId", playlet.MediaCenterId), log.Any("count", len(chapters)))
|
||||
return len(chapters), ""
|
||||
}
|
||||
|
||||
// updateSerialPlayletInfo 更新三方主资源里会随连载变化的字段
|
||||
func updateSerialPlayletInfo(ctx context.Context, playlet *playletmod.Playlet, media *proto.PlayletResourceListItem, mediaSyncTime int64) string {
|
||||
updateStatus := normalizePlayletUpdateStatus(media.UpdateStatus)
|
||||
setDoc := bson.M{
|
||||
playletmod.FN.Playlet.TotalEpisode: int64(media.TotalEpisode),
|
||||
playletmod.FN.Playlet.UpdateStatus: updateStatus,
|
||||
playletmod.FN.Playlet.MediaUpdatedAt: mediaResourceTime(media.UpdateTime),
|
||||
playletmod.FN.Playlet.UpdatedAt: time.Now(),
|
||||
}
|
||||
if mediaSyncTime > playlet.MediaSyncTime {
|
||||
setDoc[playletmod.FN.Playlet.MediaSyncTime] = mediaSyncTime
|
||||
}
|
||||
if business := playletMediaBusiness(media); business != "" && playlet.MediaBusiness != business {
|
||||
setDoc[playletmod.FN.Playlet.MediaBusiness] = business
|
||||
}
|
||||
if playName := strings.TrimSpace(playlet.PlayName); playName != "" {
|
||||
description := playletmod.BuildPlayNameDescription(playlet.Desc, playName, updateStatus, int64(media.TotalEpisode))
|
||||
if description != "" {
|
||||
setDoc[playletmod.FN.Playlet.Description] = description
|
||||
setDoc[playletmod.FN.Playlet.SeoInfo.Description] = description
|
||||
}
|
||||
}
|
||||
if err := playletmod.UpdateOnePlaylet(ctx, bson.M{
|
||||
playletmod.FN.Playlet.Id: playlet.Id,
|
||||
}, bson.M{"$set": setDoc}); err != nil {
|
||||
log.Error("更新连载短剧主表失败", log.Any("playletId", playlet.Id), log.Any("mediaCenterId", playlet.MediaCenterId), log.Any("err", err))
|
||||
return "更新主表失败"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// buildNewSerialPlayletChapters 根据本地已存在章节做幂等过滤，只返回需要追加的新章节
|
||||
func buildNewSerialPlayletChapters(ctx context.Context, playletId int64, mediaSource string, mediaContentList []playletMediaContent) (playletchaptermod.PlayletChapterList, string) {
|
||||
exists, err := loadExistingPlayletChapterKeys(ctx, playletId)
|
||||
if err != nil {
|
||||
return nil, "查询已有章节失败"
|
||||
}
|
||||
chapters := buildPlayletChaptersFromMedia(mediaContentList, mediaSource)
|
||||
result := make(playletchaptermod.PlayletChapterList, 0, len(chapters))
|
||||
for _, chapter := range chapters {
|
||||
if chapter == nil || chapterDuplicateByKeys(chapter, exists) {
|
||||
continue
|
||||
}
|
||||
chapter.PlayletId = playletId
|
||||
markExistingPlayletChapterKeys(chapter, exists)
|
||||
result = append(result, chapter)
|
||||
}
|
||||
return result, ""
|
||||
}
|
||||
|
||||
type existingPlayletChapterKeys struct {
|
||||
mediaContentIds map[uint]struct{}
|
||||
episodes map[int64]struct{}
|
||||
hashes map[string]struct{}
|
||||
videoUrls map[string]struct{}
|
||||
}
|
||||
|
||||
// loadExistingPlayletChapterKeys 加载当前短剧已入库章节的去重键，避免连载同步重复插入章节
|
||||
func loadExistingPlayletChapterKeys(ctx context.Context, playletId int64) (existingPlayletChapterKeys, error) {
|
||||
keys := existingPlayletChapterKeys{
|
||||
mediaContentIds: make(map[uint]struct{}),
|
||||
episodes: make(map[int64]struct{}),
|
||||
hashes: make(map[string]struct{}),
|
||||
videoUrls: make(map[string]struct{}),
|
||||
}
|
||||
list, serr := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{playletchaptermod.FN.PlayletChapter.PlayletId: playletId})
|
||||
if serr != nil {
|
||||
return keys, fmt.Errorf("query playlet chapters failed: %+v", serr)
|
||||
}
|
||||
for _, chapter := range list {
|
||||
markExistingPlayletChapterKeys(chapter, keys)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// chapterDuplicateByKeys 章节跨批次去重，优先用媒资章节ID，其次用集数、hash、播放地址兜底
|
||||
func chapterDuplicateByKeys(chapter *playletchaptermod.PlayletChapter, keys existingPlayletChapterKeys) bool {
|
||||
if chapter.MediaContentId > 0 {
|
||||
if _, ok := keys.mediaContentIds[chapter.MediaContentId]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if chapter.CurrentEpisode > 0 {
|
||||
if _, ok := keys.episodes[chapter.CurrentEpisode]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if chapter.Hash != "" {
|
||||
if _, ok := keys.hashes[chapter.Hash]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if chapter.VideoUrl != "" {
|
||||
if _, ok := keys.videoUrls[chapter.VideoUrl]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// markExistingPlayletChapterKeys 把已存在或本批次准备插入的章节登记到去重集合
|
||||
func markExistingPlayletChapterKeys(chapter *playletchaptermod.PlayletChapter, keys existingPlayletChapterKeys) {
|
||||
if chapter == nil {
|
||||
return
|
||||
}
|
||||
if chapter.MediaContentId > 0 {
|
||||
keys.mediaContentIds[chapter.MediaContentId] = struct{}{}
|
||||
}
|
||||
if chapter.CurrentEpisode > 0 {
|
||||
keys.episodes[chapter.CurrentEpisode] = struct{}{}
|
||||
}
|
||||
if chapter.Hash != "" {
|
||||
keys.hashes[chapter.Hash] = struct{}{}
|
||||
}
|
||||
if chapter.VideoUrl != "" {
|
||||
keys.videoUrls[chapter.VideoUrl] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// maxPlayletMediaContentUpdateTime 记录本次同步返回章节中的最大更新时间，作为下次三方增量同步点
|
||||
func maxPlayletMediaContentUpdateTime(syncTime int64, mediaContentList []playletMediaContent) int64 {
|
||||
result := syncTime
|
||||
for _, item := range mediaContentList {
|
||||
if item.UpdateTime > result {
|
||||
result = item.UpdateTime
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// normalizePlayletUpdateStatus 三方 update_status=0 在本项目按 2-已完结 保存
|
||||
func normalizePlayletUpdateStatus(updateStatus int) int {
|
||||
if updateStatus == 0 {
|
||||
return 2
|
||||
}
|
||||
return updateStatus
|
||||
}
|
||||
|
||||
// applyPlayletDefaultPayToChapters 上架且连载中时，新增章节默认沿用剧级支付设置。
|
||||
func applyPlayletDefaultPayToChapters(playlet *playletmod.Playlet, chapters playletchaptermod.PlayletChapterList) {
|
||||
if playlet == nil || len(chapters) == 0 {
|
||||
return
|
||||
}
|
||||
if playlet.Status != playletmod.StatusUp || playlet.UpdateStatus != 1 {
|
||||
return
|
||||
}
|
||||
payType := playletchaptermod.PayType(playlet.PayType)
|
||||
singleCoins := playlet.SingleCoins
|
||||
if payType != playletchaptermod.PayTypeCoin {
|
||||
singleCoins = 0
|
||||
}
|
||||
for _, chapter := range chapters {
|
||||
if chapter == nil {
|
||||
continue
|
||||
}
|
||||
chapter.PayType = payType
|
||||
chapter.SingleCoins = singleCoins
|
||||
}
|
||||
}
|
||||
|
||||
// lockPlayletSerialSync 锁定单个媒资ID，避免同一部连载短剧被多个 worker 同时同步
|
||||
func lockPlayletSerialSync(mediaId uint) (bool, error) {
|
||||
return redis.Setnx_NewOK(playletSerialSyncLockKey(mediaId), true, playletSerialSyncLockTTL)
|
||||
}
|
||||
|
||||
// unlockPlayletSerialSync 释放单个媒资ID的同步锁
|
||||
func unlockPlayletSerialSync(mediaId uint) {
|
||||
if err := redis.Del(playletSerialSyncLockKey(mediaId)); err != nil {
|
||||
log.Error("释放连载短剧同步锁失败", log.Any("mediaId", mediaId), log.E(err))
|
||||
}
|
||||
}
|
||||
|
||||
// lockPlayletSerialSyncBySource 按媒资来源锁定单个媒资ID
|
||||
func lockPlayletSerialSyncBySource(mediaSource string, mediaId uint) (bool, error) {
|
||||
if mediaSource == playletmod.MediaSourceHG {
|
||||
return lockPlayletSerialSync(mediaId)
|
||||
}
|
||||
return redis.Setnx_NewOK(playletSerialSyncJHLockKey(mediaId), true, playletSerialSyncLockTTL)
|
||||
}
|
||||
|
||||
// unlockPlayletSerialSyncBySource 按媒资来源释放单个媒资ID的同步锁
|
||||
func unlockPlayletSerialSyncBySource(mediaSource string, mediaId uint) {
|
||||
if mediaSource == playletmod.MediaSourceHG {
|
||||
unlockPlayletSerialSync(mediaId)
|
||||
return
|
||||
}
|
||||
if err := redis.Del(playletSerialSyncJHLockKey(mediaId)); err != nil {
|
||||
log.Error("释放嘉华连载短剧同步锁失败", log.Any("mediaId", mediaId), log.E(err))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user