package shortrecommend import ( "context" "fmt" "time" "91porn-server/common/log" redisutil "91porn-server/common/redis" recommendqueue "91porn-server/common/shortrecommend" "91porn-server/models/v/moduleconfmod" "91porn-server/models/v/vidmod" "91porn-server/skd/skdg" "go.mongodb.org/mongo-driver/bson/primitive" ) const ( buildLockTTL = 10 * time.Minute buildLockRenewInterval = 3 * time.Minute buildTimeout = 30 * time.Minute healthCheckTimeout = 5 * time.Second ) const renewBuildLockScriptSource = ` if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end return redis.call('EXPIRE', KEYS[1], ARGV[2]) ` const releaseBuildLockScriptSource = ` if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end return redis.call('DEL', KEYS[1]) ` var ( renewBuildLockScript = redisutil.NewScript(renewBuildLockScriptSource) releaseBuildLockScript = redisutil.NewScript(releaseBuildLockScriptSource) ) func evalLockScript( ctx context.Context, script *redisutil.Script, key, token string, args ...interface{}, ) (bool, error) { values := append([]interface{}{token}, args...) raw, err := skdg.Redis.RunScriptContext(ctx, script, []string{key}, values...) if err != nil { return false, err } return fmt.Sprint(raw) == "1", nil } func renewBuildLock(ctx context.Context, key, token string) (bool, error) { return evalLockScript( ctx, renewBuildLockScript, key, token, int64(buildLockTTL/time.Second), ) } func releaseBuildLock(ctx context.Context, key, token string) error { _, err := evalLockScript(ctx, releaseBuildLockScript, key, token) return err } func keepBuildLock(ctx context.Context, cancel context.CancelFunc, key, token, version string) { ticker := time.NewTicker(buildLockRenewInterval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: renewed, err := renewBuildLock(ctx, key, token) if err != nil || !renewed { log.Error("short recommend renew build lock failed", log.E(err), log.Any("version", version)) cancel() return } } } } func buildQueueVersion(now time.Time, buildToken string) string { return recommendqueue.VersionWithRevision(now, buildToken) } // Run 刷新真实互动分并发布当日全局短视频队列。 func Run() { now := time.Now() dateVersion := recommendqueue.VersionAt(now) healthCtx, healthCancel := context.WithTimeout(context.Background(), healthCheckTimeout) health, err := recommendqueue.HealthContext(healthCtx, skdg.Redis, dateVersion) healthCancel() if err != nil { log.Error("short recommend queue health check failed", log.E(err), log.Any("dateVersion", dateVersion)) return } if health.Healthy { return } log.Warn("short recommend queue unhealthy, start rebuild", log.Any("dateVersion", dateVersion), log.Any("currentVersion", health.Version), log.Any("healthStatus", health.Status), log.Any("queueLength", health.QueueLength), log.Any("metadataLength", health.MetadataLength), log.Any("queueTTLSeconds", int64(health.QueueTTL/time.Second)), log.Any("metadataTTLSeconds", int64(health.MetadataTTL/time.Second))) lockKey := recommendqueue.LockKey(dateVersion) lockToken := primitive.NewObjectID().Hex() lockCtx, lockCancel := context.WithTimeout(context.Background(), healthCheckTimeout) locked, err := skdg.Redis.SetNXContext(lockCtx, lockKey, lockToken, buildLockTTL) lockCancel() if err != nil { log.Error("short recommend acquire build lock failed", log.E(err), log.Any("dateVersion", dateVersion)) return } if !locked { return } buildCtx, cancel := context.WithTimeout(context.Background(), buildTimeout) lockKeeperDone := make(chan struct{}) go func() { defer close(lockKeeperDone) keepBuildLock(buildCtx, cancel, lockKey, lockToken, dateVersion) }() defer func() { cancel() <-lockKeeperDone releaseCtx, releaseCancel := context.WithTimeout(context.Background(), healthCheckTimeout) defer releaseCancel() if err := releaseBuildLock(releaseCtx, lockKey, lockToken); err != nil { log.Error("short recommend release build lock failed", log.E(err), log.Any("dateVersion", dateVersion)) } }() // 首次健康检查与加锁之间可能已有其他实例完成发布;持锁后复检避免重复全量扫描。 recheckCtx, recheckCancel := context.WithTimeout(buildCtx, healthCheckTimeout) rechecked, recheckErr := recommendqueue.HealthContext(recheckCtx, skdg.Redis, dateVersion) recheckCancel() if recheckErr != nil { log.Error("short recommend queue health recheck failed", log.E(recheckErr), log.Any("dateVersion", dateVersion)) return } if rechecked.Healthy { return } queueVersion := buildQueueVersion(now, lockToken) buildStartedAt := time.Now() excludedModuleIDs, err := moduleconfmod.ExcludedVideoModuleIDs(now, true) if err != nil { log.Error("short recommend load excluded modules failed", log.E(err), log.Any("version", queueVersion)) return } refreshStartedAt := time.Now() initializationLimit := skdg.Conf.ShortRecommendInitializationLimit() candidates, err := vidmod.RefreshRecommendScoresWithInitializationLimit( buildCtx, now, initializationLimit, excludedModuleIDs..., ) refreshDuration := time.Since(refreshStartedAt) if err != nil { log.Error("short recommend refresh scores failed", log.E(err), log.Any("version", queueVersion), log.Any("initializationWriteLimit", initializationLimit), log.Any("refreshDurationMs", refreshDuration.Milliseconds())) return } assembleStartedAt := time.Now() ids, assemblyStats := recommendqueue.AssembleWithStats(candidates, now) assembleDuration := time.Since(assembleStartedAt) if len(ids) == 0 { log.Warn("short recommend candidates empty, keep previous queue", log.Any("version", queueVersion)) return } if err := buildCtx.Err(); err != nil { log.Error("short recommend build lock lost before publish", log.E(err), log.Any("version", queueVersion)) return } if currentVersion := recommendqueue.VersionAt(time.Now()); currentVersion != dateVersion { log.Warn("short recommend build crossed version boundary, skip publish", log.Any("version", queueVersion), log.Any("currentDateVersion", currentVersion)) return } renewed, err := renewBuildLock(buildCtx, lockKey, lockToken) if err != nil || !renewed { log.Error("short recommend verify build lock failed", log.E(err), log.Any("version", queueVersion)) return } publishStartedAt := time.Now() err = recommendqueue.Publish(buildCtx, skdg.Redis, recommendqueue.QueueMeta{ Version: queueVersion, Length: len(ids), GeneratedAt: now, StartOffset: 0, }, ids, lockKey, lockToken) publishDuration := time.Since(publishStartedAt) if err != nil { log.Error("short recommend publish queue failed", log.E(err), log.Any("version", queueVersion), log.Any("publishDurationMs", publishDuration.Milliseconds())) return } log.Info("short recommend queue published", log.Any("version", queueVersion), log.Any("dateVersion", dateVersion), log.Any("candidateCount", len(candidates)), log.Any("excludedModuleCount", len(excludedModuleIDs)), log.Any("queueLength", len(ids)), log.Any("highScoreCount", assemblyStats.HighCount), log.Any("newVideoCount", assemblyStats.NewCount), log.Any("blockCount", assemblyStats.BlockCount), log.Any("refreshDurationMs", refreshDuration.Milliseconds()), log.Any("assembleDurationMs", assembleDuration.Milliseconds()), log.Any("publishDurationMs", publishDuration.Milliseconds()), log.Any("totalDurationMs", time.Since(buildStartedAt).Milliseconds())) }