Claude Opus 5
@@ -0,0 +1,373 @@
|
||||
package shortrecommendser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"91porn-server/app/appg"
|
||||
recommendqueue "91porn-server/common/shortrecommend"
|
||||
"91porn-server/models/v/moduleconfmod"
|
||||
"91porn-server/models/v/vidmod"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultMaxBatches = 5
|
||||
defaultScanMultiplier = 5
|
||||
maxAllowedBatches = 20
|
||||
maxAllowedMultiplier = 20
|
||||
maxRequestIDLength = 128
|
||||
reservationAbortTTL = 2 * time.Second
|
||||
reservationBusyRetries = 3
|
||||
reservationBusyBackoff = 10 * time.Millisecond
|
||||
)
|
||||
|
||||
// FetchResult 是一次环形队列拉取的结果和可观测指标。
|
||||
type FetchResult struct {
|
||||
Videos []*vidmod.VideoModel
|
||||
QueueVersion string
|
||||
QueueLength int
|
||||
Scanned int
|
||||
Filtered int
|
||||
Batches int
|
||||
BudgetExceeded bool
|
||||
}
|
||||
|
||||
type fetchOptions struct {
|
||||
maxBatches int
|
||||
scanMultiplier int
|
||||
}
|
||||
|
||||
type reserveFunc func(
|
||||
context.Context,
|
||||
uint64,
|
||||
int,
|
||||
string,
|
||||
) (recommendqueue.Reservation, error)
|
||||
|
||||
type fetchDependencies struct {
|
||||
reserve reserveFunc
|
||||
commit func(
|
||||
context.Context,
|
||||
uint64,
|
||||
recommendqueue.Reservation,
|
||||
int,
|
||||
) error
|
||||
abort func(
|
||||
context.Context,
|
||||
uint64,
|
||||
recommendqueue.Reservation,
|
||||
) error
|
||||
excludedModuleIDs func(time.Time, bool) ([]string, error)
|
||||
findVideos func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error)
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// Fetch 从用户当前偏移预留短视频，按当前Mongo状态过滤，并限制补位扫描成本。
|
||||
// 只有全部Mongo批次成功后才一次性推进实际扫描前缀；失败会释放预留且不跳内容。
|
||||
func Fetch(
|
||||
ctx context.Context,
|
||||
uid uint64,
|
||||
size int,
|
||||
requestID string,
|
||||
) (FetchResult, error) {
|
||||
return FetchScoped(ctx, uid, size, requestID, "short-recommend")
|
||||
}
|
||||
|
||||
// FetchScoped 将入口和请求大小纳入幂等作用域，防止同一客户端requestID
|
||||
// 被不同短视频接口或不同请求参数误复用。
|
||||
func FetchScoped(
|
||||
ctx context.Context,
|
||||
uid uint64,
|
||||
size int,
|
||||
requestID, scope string,
|
||||
) (FetchResult, error) {
|
||||
if appg.Redis == nil {
|
||||
return FetchResult{}, fmt.Errorf("short recommend Redis is nil")
|
||||
}
|
||||
var err error
|
||||
requestID, err = scopedRequestID(scope, size, requestID)
|
||||
if err != nil {
|
||||
return FetchResult{}, err
|
||||
}
|
||||
opts := configuredFetchOptions()
|
||||
deps := fetchDependencies{
|
||||
reserve: func(
|
||||
ctx context.Context,
|
||||
uid uint64,
|
||||
size int,
|
||||
requestID string,
|
||||
) (recommendqueue.Reservation, error) {
|
||||
return recommendqueue.ReserveCurrentContext(
|
||||
ctx, appg.Redis, uid, size, requestID,
|
||||
)
|
||||
},
|
||||
commit: func(
|
||||
ctx context.Context,
|
||||
uid uint64,
|
||||
reservation recommendqueue.Reservation,
|
||||
consumed int,
|
||||
) error {
|
||||
return recommendqueue.CommitReservationContext(
|
||||
ctx, appg.Redis, uid, reservation, consumed,
|
||||
)
|
||||
},
|
||||
abort: func(
|
||||
ctx context.Context,
|
||||
uid uint64,
|
||||
reservation recommendqueue.Reservation,
|
||||
) error {
|
||||
return recommendqueue.AbortReservationContext(
|
||||
ctx, appg.Redis, uid, reservation,
|
||||
)
|
||||
},
|
||||
excludedModuleIDs: moduleconfmod.ExcludedVideoModuleIDs,
|
||||
findVideos: vidmod.GetRecommendVideosByIDsContext,
|
||||
now: time.Now,
|
||||
}
|
||||
return fetch(ctx, uid, size, requestID, opts, deps)
|
||||
}
|
||||
|
||||
func scopedRequestID(scope string, size int, requestID string) (string, error) {
|
||||
requestID = strings.TrimSpace(requestID)
|
||||
if requestID == "" {
|
||||
return "", nil
|
||||
}
|
||||
if len(requestID) > maxRequestIDLength {
|
||||
return "", fmt.Errorf("short recommend request ID exceeds %d bytes", maxRequestIDLength)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(fmt.Sprintf("%s:%d:%s", scope, size, requestID)))
|
||||
return fmt.Sprintf("%x", sum[:16]), nil
|
||||
}
|
||||
|
||||
func configuredFetchOptions() fetchOptions {
|
||||
opts := fetchOptions{
|
||||
maxBatches: defaultMaxBatches,
|
||||
scanMultiplier: defaultScanMultiplier,
|
||||
}
|
||||
if appg.Conf == nil {
|
||||
return opts
|
||||
}
|
||||
if configured := appg.Conf.ShortRecommend.MaxBatches; configured > 0 {
|
||||
opts.maxBatches = clamp(configured, 1, maxAllowedBatches)
|
||||
}
|
||||
if configured := appg.Conf.ShortRecommend.ScanMultiplier; configured > 0 {
|
||||
opts.scanMultiplier = clamp(configured, 1, maxAllowedMultiplier)
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func fetch(
|
||||
ctx context.Context,
|
||||
uid uint64,
|
||||
size int,
|
||||
requestID string,
|
||||
opts fetchOptions,
|
||||
deps fetchDependencies,
|
||||
) (result FetchResult, err error) {
|
||||
if ctx == nil {
|
||||
return result, fmt.Errorf("short recommend context must not be nil")
|
||||
}
|
||||
if err = ctx.Err(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if uid == 0 {
|
||||
return result, fmt.Errorf("anonymous user has no independent queue offset")
|
||||
}
|
||||
if size <= 0 {
|
||||
return result, nil
|
||||
}
|
||||
requestID = strings.TrimSpace(requestID)
|
||||
if len(requestID) > maxRequestIDLength {
|
||||
return result, fmt.Errorf("short recommend request ID exceeds %d bytes", maxRequestIDLength)
|
||||
}
|
||||
opts.maxBatches = clamp(opts.maxBatches, 1, maxAllowedBatches)
|
||||
opts.scanMultiplier = clamp(opts.scanMultiplier, 1, maxAllowedMultiplier)
|
||||
scanBudget := size * opts.scanMultiplier
|
||||
if scanBudget < size {
|
||||
scanBudget = size
|
||||
}
|
||||
if batchBudget := size * opts.maxBatches; scanBudget > batchBudget {
|
||||
// 预留不会超过本次最多能查询的范围，避免把永远不会扫描的ID
|
||||
// 从Redis传到App进程。
|
||||
scanBudget = batchBudget
|
||||
}
|
||||
|
||||
excluded, err := deps.excludedModuleIDs(deps.now(), true)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
excludedSet := make(map[string]struct{}, len(excluded))
|
||||
for _, moduleID := range excluded {
|
||||
excludedSet[moduleID] = struct{}{}
|
||||
}
|
||||
|
||||
reservation, reserveErr := reserveWithBusyRetry(
|
||||
ctx, deps.reserve, uid, scanBudget, requestID,
|
||||
)
|
||||
result.QueueVersion = reservation.Version
|
||||
result.QueueLength = reservation.Length
|
||||
if reserveErr != nil {
|
||||
return result, reserveErr
|
||||
}
|
||||
if reservation.Version == "" || reservation.Length <= 0 ||
|
||||
reservation.Reserved <= 0 || len(reservation.IDs) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
committed := false
|
||||
defer func() {
|
||||
if committed {
|
||||
return
|
||||
}
|
||||
abortCtx, cancel := context.WithTimeout(context.Background(), reservationAbortTTL)
|
||||
defer cancel()
|
||||
_ = deps.abort(abortCtx, uid, reservation)
|
||||
}()
|
||||
|
||||
result.Videos = make([]*vidmod.VideoModel, 0, size)
|
||||
seen := make(map[primitive.ObjectID]struct{}, size)
|
||||
cursor := 0
|
||||
for cursor < len(reservation.IDs) &&
|
||||
(reservation.AlreadyCommitted ||
|
||||
(result.Batches < opts.maxBatches && len(result.Videos) < size)) {
|
||||
batchSize := size
|
||||
if !reservation.AlreadyCommitted {
|
||||
batchSize = size - len(result.Videos)
|
||||
} else if batchSize > 0 {
|
||||
// 已提交请求的receipt记录的是原实际消费前缀。重试必须扫描并
|
||||
// 确认完整前缀，再以原consumed幂等提交，不能因当前状态变化
|
||||
// 缩短已确认的消费范围。固定按请求大小分批，避免当前只差一条
|
||||
// 有效视频时退化为大量单ID Mongo查询。
|
||||
}
|
||||
if remaining := len(reservation.IDs) - cursor; batchSize > remaining {
|
||||
batchSize = remaining
|
||||
}
|
||||
if batchSize <= 0 {
|
||||
break
|
||||
}
|
||||
end := cursor + batchSize
|
||||
batchIDs := reservation.IDs[cursor:end]
|
||||
objectIDs := parseUniqueObjectIDs(batchIDs)
|
||||
videos, findErr := deps.findVideos(ctx, objectIDs)
|
||||
if findErr != nil {
|
||||
result.Videos = nil
|
||||
return result, findErr
|
||||
}
|
||||
result.Batches++
|
||||
result.Scanned += len(batchIDs)
|
||||
cursor = end
|
||||
byID := make(map[primitive.ObjectID]*vidmod.VideoModel, len(videos))
|
||||
for _, video := range videos {
|
||||
if video == nil {
|
||||
continue
|
||||
}
|
||||
if _, blocked := excludedSet[video.MID]; blocked {
|
||||
continue
|
||||
}
|
||||
byID[video.ID] = video
|
||||
}
|
||||
before := len(result.Videos)
|
||||
for _, rawID := range batchIDs {
|
||||
if len(result.Videos) >= size {
|
||||
break
|
||||
}
|
||||
id, parseErr := primitive.ObjectIDFromHex(rawID)
|
||||
if parseErr != nil {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[id]; exists {
|
||||
continue
|
||||
}
|
||||
video := byID[id]
|
||||
if video == nil {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
result.Videos = append(result.Videos, video)
|
||||
if len(result.Videos) == size {
|
||||
break
|
||||
}
|
||||
}
|
||||
result.Filtered += len(batchIDs) - (len(result.Videos) - before)
|
||||
}
|
||||
if result.Scanned <= 0 {
|
||||
result.Videos = nil
|
||||
return result, fmt.Errorf("short recommend reservation was not scanned")
|
||||
}
|
||||
commitErr := deps.commit(ctx, uid, reservation, result.Scanned)
|
||||
if commitErr != nil && ctx.Err() == nil {
|
||||
// 同一receipt重试可确认“脚本已执行但响应丢失”的不确定提交，
|
||||
// 不会再次推进offset。
|
||||
commitErr = deps.commit(ctx, uid, reservation, result.Scanned)
|
||||
}
|
||||
if commitErr != nil {
|
||||
result.Videos = nil
|
||||
return result, commitErr
|
||||
}
|
||||
committed = true
|
||||
result.BudgetExceeded = len(result.Videos) < size &&
|
||||
result.QueueLength > 0 &&
|
||||
result.Scanned < result.QueueLength &&
|
||||
(result.Scanned >= scanBudget || result.Batches >= opts.maxBatches)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func reserveWithBusyRetry(
|
||||
ctx context.Context,
|
||||
reserve reserveFunc,
|
||||
uid uint64,
|
||||
size int,
|
||||
requestID string,
|
||||
) (recommendqueue.Reservation, error) {
|
||||
var reservation recommendqueue.Reservation
|
||||
var err error
|
||||
for attempt := 0; attempt <= reservationBusyRetries; attempt++ {
|
||||
reservation, err = reserve(ctx, uid, size, requestID)
|
||||
if !errors.Is(err, recommendqueue.ErrReservationBusy) ||
|
||||
attempt == reservationBusyRetries {
|
||||
return reservation, err
|
||||
}
|
||||
delay := reservationBusyBackoff << attempt
|
||||
timer := time.NewTimer(delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return reservation, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return reservation, err
|
||||
}
|
||||
|
||||
func parseUniqueObjectIDs(ids []string) []primitive.ObjectID {
|
||||
out := make([]primitive.ObjectID, 0, len(ids))
|
||||
seen := make(map[primitive.ObjectID]struct{}, len(ids))
|
||||
for _, rawID := range ids {
|
||||
id, err := primitive.ObjectIDFromHex(rawID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[id]; exists {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
out = append(out, id)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func clamp(value, low, high int) int {
|
||||
if value < low {
|
||||
return low
|
||||
}
|
||||
if value > high {
|
||||
return high
|
||||
}
|
||||
return value
|
||||
}
|
||||
Reference in New Issue
Block a user