package shortrecommendser import ( "context" "crypto/sha256" "errors" "fmt" "strings" "time" "91porn-server/app/appg" recommendqueue "91porn-server/common/shortrecommend" "91porn-server/models/v/moduleconfmod" "91porn-server/models/v/vidmod" "go.mongodb.org/mongo-driver/bson/primitive" ) const ( defaultMaxBatches = 5 defaultScanMultiplier = 5 maxAllowedBatches = 20 maxAllowedMultiplier = 20 maxRequestIDLength = 128 reservationAbortTTL = 2 * time.Second reservationBusyRetries = 3 reservationBusyBackoff = 10 * time.Millisecond ) // FetchResult 是一次环形队列拉取的结果和可观测指标。 type FetchResult struct { Videos []*vidmod.VideoModel QueueVersion string QueueLength int Scanned int Filtered int Batches int BudgetExceeded bool } type fetchOptions struct { maxBatches int scanMultiplier int } type reserveFunc func( context.Context, uint64, int, string, ) (recommendqueue.Reservation, error) type fetchDependencies struct { reserve reserveFunc commit func( context.Context, uint64, recommendqueue.Reservation, int, ) error abort func( context.Context, uint64, recommendqueue.Reservation, ) error excludedModuleIDs func(time.Time, bool) ([]string, error) findVideos func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) now func() time.Time } // Fetch 从用户当前偏移预留短视频,按当前Mongo状态过滤,并限制补位扫描成本。 // 只有全部Mongo批次成功后才一次性推进实际扫描前缀;失败会释放预留且不跳内容。 func Fetch( ctx context.Context, uid uint64, size int, requestID string, ) (FetchResult, error) { return FetchScoped(ctx, uid, size, requestID, "short-recommend") } // FetchScoped 将入口和请求大小纳入幂等作用域,防止同一客户端requestID // 被不同短视频接口或不同请求参数误复用。 func FetchScoped( ctx context.Context, uid uint64, size int, requestID, scope string, ) (FetchResult, error) { if appg.Redis == nil { return FetchResult{}, fmt.Errorf("short recommend Redis is nil") } var err error requestID, err = scopedRequestID(scope, size, requestID) if err != nil { return FetchResult{}, err } opts := configuredFetchOptions() deps := fetchDependencies{ reserve: func( ctx context.Context, uid uint64, size int, requestID string, ) (recommendqueue.Reservation, error) { return recommendqueue.ReserveCurrentContext( ctx, appg.Redis, uid, size, requestID, ) }, commit: func( ctx context.Context, uid uint64, reservation recommendqueue.Reservation, consumed int, ) error { return recommendqueue.CommitReservationContext( ctx, appg.Redis, uid, reservation, consumed, ) }, abort: func( ctx context.Context, uid uint64, reservation recommendqueue.Reservation, ) error { return recommendqueue.AbortReservationContext( ctx, appg.Redis, uid, reservation, ) }, excludedModuleIDs: moduleconfmod.ExcludedVideoModuleIDs, findVideos: vidmod.GetRecommendVideosByIDsContext, now: time.Now, } return fetch(ctx, uid, size, requestID, opts, deps) } func scopedRequestID(scope string, size int, requestID string) (string, error) { requestID = strings.TrimSpace(requestID) if requestID == "" { return "", nil } if len(requestID) > maxRequestIDLength { return "", fmt.Errorf("short recommend request ID exceeds %d bytes", maxRequestIDLength) } sum := sha256.Sum256([]byte(fmt.Sprintf("%s:%d:%s", scope, size, requestID))) return fmt.Sprintf("%x", sum[:16]), nil } func configuredFetchOptions() fetchOptions { opts := fetchOptions{ maxBatches: defaultMaxBatches, scanMultiplier: defaultScanMultiplier, } if appg.Conf == nil { return opts } if configured := appg.Conf.ShortRecommend.MaxBatches; configured > 0 { opts.maxBatches = clamp(configured, 1, maxAllowedBatches) } if configured := appg.Conf.ShortRecommend.ScanMultiplier; configured > 0 { opts.scanMultiplier = clamp(configured, 1, maxAllowedMultiplier) } return opts } func fetch( ctx context.Context, uid uint64, size int, requestID string, opts fetchOptions, deps fetchDependencies, ) (result FetchResult, err error) { if ctx == nil { return result, fmt.Errorf("short recommend context must not be nil") } if err = ctx.Err(); err != nil { return result, err } if uid == 0 { return result, fmt.Errorf("anonymous user has no independent queue offset") } if size <= 0 { return result, nil } requestID = strings.TrimSpace(requestID) if len(requestID) > maxRequestIDLength { return result, fmt.Errorf("short recommend request ID exceeds %d bytes", maxRequestIDLength) } opts.maxBatches = clamp(opts.maxBatches, 1, maxAllowedBatches) opts.scanMultiplier = clamp(opts.scanMultiplier, 1, maxAllowedMultiplier) scanBudget := size * opts.scanMultiplier if scanBudget < size { scanBudget = size } if batchBudget := size * opts.maxBatches; scanBudget > batchBudget { // 预留不会超过本次最多能查询的范围,避免把永远不会扫描的ID // 从Redis传到App进程。 scanBudget = batchBudget } excluded, err := deps.excludedModuleIDs(deps.now(), true) if err != nil { return result, err } excludedSet := make(map[string]struct{}, len(excluded)) for _, moduleID := range excluded { excludedSet[moduleID] = struct{}{} } reservation, reserveErr := reserveWithBusyRetry( ctx, deps.reserve, uid, scanBudget, requestID, ) result.QueueVersion = reservation.Version result.QueueLength = reservation.Length if reserveErr != nil { return result, reserveErr } if reservation.Version == "" || reservation.Length <= 0 || reservation.Reserved <= 0 || len(reservation.IDs) == 0 { return result, nil } committed := false defer func() { if committed { return } abortCtx, cancel := context.WithTimeout(context.Background(), reservationAbortTTL) defer cancel() _ = deps.abort(abortCtx, uid, reservation) }() result.Videos = make([]*vidmod.VideoModel, 0, size) seen := make(map[primitive.ObjectID]struct{}, size) cursor := 0 for cursor < len(reservation.IDs) && (reservation.AlreadyCommitted || (result.Batches < opts.maxBatches && len(result.Videos) < size)) { batchSize := size if !reservation.AlreadyCommitted { batchSize = size - len(result.Videos) } else if batchSize > 0 { // 已提交请求的receipt记录的是原实际消费前缀。重试必须扫描并 // 确认完整前缀,再以原consumed幂等提交,不能因当前状态变化 // 缩短已确认的消费范围。固定按请求大小分批,避免当前只差一条 // 有效视频时退化为大量单ID Mongo查询。 } if remaining := len(reservation.IDs) - cursor; batchSize > remaining { batchSize = remaining } if batchSize <= 0 { break } end := cursor + batchSize batchIDs := reservation.IDs[cursor:end] objectIDs := parseUniqueObjectIDs(batchIDs) videos, findErr := deps.findVideos(ctx, objectIDs) if findErr != nil { result.Videos = nil return result, findErr } result.Batches++ result.Scanned += len(batchIDs) cursor = end byID := make(map[primitive.ObjectID]*vidmod.VideoModel, len(videos)) for _, video := range videos { if video == nil { continue } if _, blocked := excludedSet[video.MID]; blocked { continue } byID[video.ID] = video } before := len(result.Videos) for _, rawID := range batchIDs { if len(result.Videos) >= size { break } id, parseErr := primitive.ObjectIDFromHex(rawID) if parseErr != nil { continue } if _, exists := seen[id]; exists { continue } video := byID[id] if video == nil { continue } seen[id] = struct{}{} result.Videos = append(result.Videos, video) if len(result.Videos) == size { break } } result.Filtered += len(batchIDs) - (len(result.Videos) - before) } if result.Scanned <= 0 { result.Videos = nil return result, fmt.Errorf("short recommend reservation was not scanned") } commitErr := deps.commit(ctx, uid, reservation, result.Scanned) if commitErr != nil && ctx.Err() == nil { // 同一receipt重试可确认“脚本已执行但响应丢失”的不确定提交, // 不会再次推进offset。 commitErr = deps.commit(ctx, uid, reservation, result.Scanned) } if commitErr != nil { result.Videos = nil return result, commitErr } committed = true result.BudgetExceeded = len(result.Videos) < size && result.QueueLength > 0 && result.Scanned < result.QueueLength && (result.Scanned >= scanBudget || result.Batches >= opts.maxBatches) return result, nil } func reserveWithBusyRetry( ctx context.Context, reserve reserveFunc, uid uint64, size int, requestID string, ) (recommendqueue.Reservation, error) { var reservation recommendqueue.Reservation var err error for attempt := 0; attempt <= reservationBusyRetries; attempt++ { reservation, err = reserve(ctx, uid, size, requestID) if !errors.Is(err, recommendqueue.ErrReservationBusy) || attempt == reservationBusyRetries { return reservation, err } delay := reservationBusyBackoff << attempt timer := time.NewTimer(delay) select { case <-ctx.Done(): timer.Stop() return reservation, ctx.Err() case <-timer.C: } } return reservation, err } func parseUniqueObjectIDs(ids []string) []primitive.ObjectID { out := make([]primitive.ObjectID, 0, len(ids)) seen := make(map[primitive.ObjectID]struct{}, len(ids)) for _, rawID := range ids { id, err := primitive.ObjectIDFromHex(rawID) if err != nil { continue } if _, exists := seen[id]; exists { continue } seen[id] = struct{}{} out = append(out, id) } return out } func clamp(value, low, high int) int { if value < low { return low } if value > high { return high } return value }