Claude Opus 5
@@ -0,0 +1,231 @@
|
||||
package shortrecommend
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"91porn-server/common/log"
|
||||
redisutil "91porn-server/common/redis"
|
||||
recommendqueue "91porn-server/common/shortrecommend"
|
||||
"91porn-server/models/v/moduleconfmod"
|
||||
"91porn-server/models/v/vidmod"
|
||||
"91porn-server/skd/skdg"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
)
|
||||
|
||||
const (
|
||||
buildLockTTL = 10 * time.Minute
|
||||
buildLockRenewInterval = 3 * time.Minute
|
||||
buildTimeout = 30 * time.Minute
|
||||
healthCheckTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
const renewBuildLockScriptSource = `
|
||||
if redis.call('GET', KEYS[1]) ~= ARGV[1] then
|
||||
return 0
|
||||
end
|
||||
return redis.call('EXPIRE', KEYS[1], ARGV[2])
|
||||
`
|
||||
|
||||
const releaseBuildLockScriptSource = `
|
||||
if redis.call('GET', KEYS[1]) ~= ARGV[1] then
|
||||
return 0
|
||||
end
|
||||
return redis.call('DEL', KEYS[1])
|
||||
`
|
||||
|
||||
var (
|
||||
renewBuildLockScript = redisutil.NewScript(renewBuildLockScriptSource)
|
||||
releaseBuildLockScript = redisutil.NewScript(releaseBuildLockScriptSource)
|
||||
)
|
||||
|
||||
func evalLockScript(
|
||||
ctx context.Context,
|
||||
script *redisutil.Script,
|
||||
key, token string,
|
||||
args ...interface{},
|
||||
) (bool, error) {
|
||||
values := append([]interface{}{token}, args...)
|
||||
raw, err := skdg.Redis.RunScriptContext(ctx, script, []string{key}, values...)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return fmt.Sprint(raw) == "1", nil
|
||||
}
|
||||
|
||||
func renewBuildLock(ctx context.Context, key, token string) (bool, error) {
|
||||
return evalLockScript(
|
||||
ctx,
|
||||
renewBuildLockScript,
|
||||
key,
|
||||
token,
|
||||
int64(buildLockTTL/time.Second),
|
||||
)
|
||||
}
|
||||
|
||||
func releaseBuildLock(ctx context.Context, key, token string) error {
|
||||
_, err := evalLockScript(ctx, releaseBuildLockScript, key, token)
|
||||
return err
|
||||
}
|
||||
|
||||
func keepBuildLock(ctx context.Context, cancel context.CancelFunc, key, token, version string) {
|
||||
ticker := time.NewTicker(buildLockRenewInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
renewed, err := renewBuildLock(ctx, key, token)
|
||||
if err != nil || !renewed {
|
||||
log.Error("short recommend renew build lock failed",
|
||||
log.E(err), log.Any("version", version))
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func buildQueueVersion(now time.Time, buildToken string) string {
|
||||
return recommendqueue.VersionWithRevision(now, buildToken)
|
||||
}
|
||||
|
||||
// Run 刷新真实互动分并发布当日全局短视频队列。
|
||||
func Run() {
|
||||
now := time.Now()
|
||||
dateVersion := recommendqueue.VersionAt(now)
|
||||
healthCtx, healthCancel := context.WithTimeout(context.Background(), healthCheckTimeout)
|
||||
health, err := recommendqueue.HealthContext(healthCtx, skdg.Redis, dateVersion)
|
||||
healthCancel()
|
||||
if err != nil {
|
||||
log.Error("short recommend queue health check failed",
|
||||
log.E(err), log.Any("dateVersion", dateVersion))
|
||||
return
|
||||
}
|
||||
if health.Healthy {
|
||||
return
|
||||
}
|
||||
log.Warn("short recommend queue unhealthy, start rebuild",
|
||||
log.Any("dateVersion", dateVersion),
|
||||
log.Any("currentVersion", health.Version),
|
||||
log.Any("healthStatus", health.Status),
|
||||
log.Any("queueLength", health.QueueLength),
|
||||
log.Any("metadataLength", health.MetadataLength),
|
||||
log.Any("queueTTLSeconds", int64(health.QueueTTL/time.Second)),
|
||||
log.Any("metadataTTLSeconds", int64(health.MetadataTTL/time.Second)))
|
||||
lockKey := recommendqueue.LockKey(dateVersion)
|
||||
lockToken := primitive.NewObjectID().Hex()
|
||||
lockCtx, lockCancel := context.WithTimeout(context.Background(), healthCheckTimeout)
|
||||
locked, err := skdg.Redis.SetNXContext(lockCtx, lockKey, lockToken, buildLockTTL)
|
||||
lockCancel()
|
||||
if err != nil {
|
||||
log.Error("short recommend acquire build lock failed",
|
||||
log.E(err), log.Any("dateVersion", dateVersion))
|
||||
return
|
||||
}
|
||||
if !locked {
|
||||
return
|
||||
}
|
||||
buildCtx, cancel := context.WithTimeout(context.Background(), buildTimeout)
|
||||
lockKeeperDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(lockKeeperDone)
|
||||
keepBuildLock(buildCtx, cancel, lockKey, lockToken, dateVersion)
|
||||
}()
|
||||
defer func() {
|
||||
cancel()
|
||||
<-lockKeeperDone
|
||||
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), healthCheckTimeout)
|
||||
defer releaseCancel()
|
||||
if err := releaseBuildLock(releaseCtx, lockKey, lockToken); err != nil {
|
||||
log.Error("short recommend release build lock failed",
|
||||
log.E(err), log.Any("dateVersion", dateVersion))
|
||||
}
|
||||
}()
|
||||
|
||||
// 首次健康检查与加锁之间可能已有其他实例完成发布；持锁后复检避免重复全量扫描。
|
||||
recheckCtx, recheckCancel := context.WithTimeout(buildCtx, healthCheckTimeout)
|
||||
rechecked, recheckErr := recommendqueue.HealthContext(recheckCtx, skdg.Redis, dateVersion)
|
||||
recheckCancel()
|
||||
if recheckErr != nil {
|
||||
log.Error("short recommend queue health recheck failed",
|
||||
log.E(recheckErr), log.Any("dateVersion", dateVersion))
|
||||
return
|
||||
}
|
||||
if rechecked.Healthy {
|
||||
return
|
||||
}
|
||||
|
||||
queueVersion := buildQueueVersion(now, lockToken)
|
||||
buildStartedAt := time.Now()
|
||||
excludedModuleIDs, err := moduleconfmod.ExcludedVideoModuleIDs(now, true)
|
||||
if err != nil {
|
||||
log.Error("short recommend load excluded modules failed",
|
||||
log.E(err), log.Any("version", queueVersion))
|
||||
return
|
||||
}
|
||||
refreshStartedAt := time.Now()
|
||||
initializationLimit := skdg.Conf.ShortRecommendInitializationLimit()
|
||||
candidates, err := vidmod.RefreshRecommendScoresWithInitializationLimit(
|
||||
buildCtx, now, initializationLimit, excludedModuleIDs...,
|
||||
)
|
||||
refreshDuration := time.Since(refreshStartedAt)
|
||||
if err != nil {
|
||||
log.Error("short recommend refresh scores failed",
|
||||
log.E(err), log.Any("version", queueVersion),
|
||||
log.Any("initializationWriteLimit", initializationLimit),
|
||||
log.Any("refreshDurationMs", refreshDuration.Milliseconds()))
|
||||
return
|
||||
}
|
||||
assembleStartedAt := time.Now()
|
||||
ids, assemblyStats := recommendqueue.AssembleWithStats(candidates, now)
|
||||
assembleDuration := time.Since(assembleStartedAt)
|
||||
if len(ids) == 0 {
|
||||
log.Warn("short recommend candidates empty, keep previous queue",
|
||||
log.Any("version", queueVersion))
|
||||
return
|
||||
}
|
||||
if err := buildCtx.Err(); err != nil {
|
||||
log.Error("short recommend build lock lost before publish",
|
||||
log.E(err), log.Any("version", queueVersion))
|
||||
return
|
||||
}
|
||||
if currentVersion := recommendqueue.VersionAt(time.Now()); currentVersion != dateVersion {
|
||||
log.Warn("short recommend build crossed version boundary, skip publish",
|
||||
log.Any("version", queueVersion), log.Any("currentDateVersion", currentVersion))
|
||||
return
|
||||
}
|
||||
renewed, err := renewBuildLock(buildCtx, lockKey, lockToken)
|
||||
if err != nil || !renewed {
|
||||
log.Error("short recommend verify build lock failed",
|
||||
log.E(err), log.Any("version", queueVersion))
|
||||
return
|
||||
}
|
||||
publishStartedAt := time.Now()
|
||||
err = recommendqueue.Publish(buildCtx, skdg.Redis, recommendqueue.QueueMeta{
|
||||
Version: queueVersion, Length: len(ids), GeneratedAt: now, StartOffset: 0,
|
||||
}, ids, lockKey, lockToken)
|
||||
publishDuration := time.Since(publishStartedAt)
|
||||
if err != nil {
|
||||
log.Error("short recommend publish queue failed",
|
||||
log.E(err), log.Any("version", queueVersion),
|
||||
log.Any("publishDurationMs", publishDuration.Milliseconds()))
|
||||
return
|
||||
}
|
||||
log.Info("short recommend queue published",
|
||||
log.Any("version", queueVersion),
|
||||
log.Any("dateVersion", dateVersion),
|
||||
log.Any("candidateCount", len(candidates)),
|
||||
log.Any("excludedModuleCount", len(excludedModuleIDs)),
|
||||
log.Any("queueLength", len(ids)),
|
||||
log.Any("highScoreCount", assemblyStats.HighCount),
|
||||
log.Any("newVideoCount", assemblyStats.NewCount),
|
||||
log.Any("blockCount", assemblyStats.BlockCount),
|
||||
log.Any("refreshDurationMs", refreshDuration.Milliseconds()),
|
||||
log.Any("assembleDurationMs", assembleDuration.Milliseconds()),
|
||||
log.Any("publishDurationMs", publishDuration.Milliseconds()),
|
||||
log.Any("totalDurationMs", time.Since(buildStartedAt).Milliseconds()))
|
||||
}
|
||||
Reference in New Issue
Block a user