374 lines
9.7 KiB
Go
374 lines
9.7 KiB
Go
package shortrecommendser
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"91porn-server/app/appg"
|
|
recommendqueue "91porn-server/common/shortrecommend"
|
|
"91porn-server/models/v/moduleconfmod"
|
|
"91porn-server/models/v/vidmod"
|
|
|
|
"go.mongodb.org/mongo-driver/bson/primitive"
|
|
)
|
|
|
|
const (
|
|
defaultMaxBatches = 5
|
|
defaultScanMultiplier = 5
|
|
maxAllowedBatches = 20
|
|
maxAllowedMultiplier = 20
|
|
maxRequestIDLength = 128
|
|
reservationAbortTTL = 2 * time.Second
|
|
reservationBusyRetries = 3
|
|
reservationBusyBackoff = 10 * time.Millisecond
|
|
)
|
|
|
|
// FetchResult 是一次环形队列拉取的结果和可观测指标。
|
|
type FetchResult struct {
|
|
Videos []*vidmod.VideoModel
|
|
QueueVersion string
|
|
QueueLength int
|
|
Scanned int
|
|
Filtered int
|
|
Batches int
|
|
BudgetExceeded bool
|
|
}
|
|
|
|
type fetchOptions struct {
|
|
maxBatches int
|
|
scanMultiplier int
|
|
}
|
|
|
|
type reserveFunc func(
|
|
context.Context,
|
|
uint64,
|
|
int,
|
|
string,
|
|
) (recommendqueue.Reservation, error)
|
|
|
|
type fetchDependencies struct {
|
|
reserve reserveFunc
|
|
commit func(
|
|
context.Context,
|
|
uint64,
|
|
recommendqueue.Reservation,
|
|
int,
|
|
) error
|
|
abort func(
|
|
context.Context,
|
|
uint64,
|
|
recommendqueue.Reservation,
|
|
) error
|
|
excludedModuleIDs func(time.Time, bool) ([]string, error)
|
|
findVideos func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error)
|
|
now func() time.Time
|
|
}
|
|
|
|
// Fetch 从用户当前偏移预留短视频，按当前Mongo状态过滤，并限制补位扫描成本。
|
|
// 只有全部Mongo批次成功后才一次性推进实际扫描前缀；失败会释放预留且不跳内容。
|
|
func Fetch(
|
|
ctx context.Context,
|
|
uid uint64,
|
|
size int,
|
|
requestID string,
|
|
) (FetchResult, error) {
|
|
return FetchScoped(ctx, uid, size, requestID, "short-recommend")
|
|
}
|
|
|
|
// FetchScoped 将入口和请求大小纳入幂等作用域，防止同一客户端requestID
|
|
// 被不同短视频接口或不同请求参数误复用。
|
|
func FetchScoped(
|
|
ctx context.Context,
|
|
uid uint64,
|
|
size int,
|
|
requestID, scope string,
|
|
) (FetchResult, error) {
|
|
if appg.Redis == nil {
|
|
return FetchResult{}, fmt.Errorf("short recommend Redis is nil")
|
|
}
|
|
var err error
|
|
requestID, err = scopedRequestID(scope, size, requestID)
|
|
if err != nil {
|
|
return FetchResult{}, err
|
|
}
|
|
opts := configuredFetchOptions()
|
|
deps := fetchDependencies{
|
|
reserve: func(
|
|
ctx context.Context,
|
|
uid uint64,
|
|
size int,
|
|
requestID string,
|
|
) (recommendqueue.Reservation, error) {
|
|
return recommendqueue.ReserveCurrentContext(
|
|
ctx, appg.Redis, uid, size, requestID,
|
|
)
|
|
},
|
|
commit: func(
|
|
ctx context.Context,
|
|
uid uint64,
|
|
reservation recommendqueue.Reservation,
|
|
consumed int,
|
|
) error {
|
|
return recommendqueue.CommitReservationContext(
|
|
ctx, appg.Redis, uid, reservation, consumed,
|
|
)
|
|
},
|
|
abort: func(
|
|
ctx context.Context,
|
|
uid uint64,
|
|
reservation recommendqueue.Reservation,
|
|
) error {
|
|
return recommendqueue.AbortReservationContext(
|
|
ctx, appg.Redis, uid, reservation,
|
|
)
|
|
},
|
|
excludedModuleIDs: moduleconfmod.ExcludedVideoModuleIDs,
|
|
findVideos: vidmod.GetRecommendVideosByIDsContext,
|
|
now: time.Now,
|
|
}
|
|
return fetch(ctx, uid, size, requestID, opts, deps)
|
|
}
|
|
|
|
func scopedRequestID(scope string, size int, requestID string) (string, error) {
|
|
requestID = strings.TrimSpace(requestID)
|
|
if requestID == "" {
|
|
return "", nil
|
|
}
|
|
if len(requestID) > maxRequestIDLength {
|
|
return "", fmt.Errorf("short recommend request ID exceeds %d bytes", maxRequestIDLength)
|
|
}
|
|
sum := sha256.Sum256([]byte(fmt.Sprintf("%s:%d:%s", scope, size, requestID)))
|
|
return fmt.Sprintf("%x", sum[:16]), nil
|
|
}
|
|
|
|
func configuredFetchOptions() fetchOptions {
|
|
opts := fetchOptions{
|
|
maxBatches: defaultMaxBatches,
|
|
scanMultiplier: defaultScanMultiplier,
|
|
}
|
|
if appg.Conf == nil {
|
|
return opts
|
|
}
|
|
if configured := appg.Conf.ShortRecommend.MaxBatches; configured > 0 {
|
|
opts.maxBatches = clamp(configured, 1, maxAllowedBatches)
|
|
}
|
|
if configured := appg.Conf.ShortRecommend.ScanMultiplier; configured > 0 {
|
|
opts.scanMultiplier = clamp(configured, 1, maxAllowedMultiplier)
|
|
}
|
|
return opts
|
|
}
|
|
|
|
func fetch(
|
|
ctx context.Context,
|
|
uid uint64,
|
|
size int,
|
|
requestID string,
|
|
opts fetchOptions,
|
|
deps fetchDependencies,
|
|
) (result FetchResult, err error) {
|
|
if ctx == nil {
|
|
return result, fmt.Errorf("short recommend context must not be nil")
|
|
}
|
|
if err = ctx.Err(); err != nil {
|
|
return result, err
|
|
}
|
|
if uid == 0 {
|
|
return result, fmt.Errorf("anonymous user has no independent queue offset")
|
|
}
|
|
if size <= 0 {
|
|
return result, nil
|
|
}
|
|
requestID = strings.TrimSpace(requestID)
|
|
if len(requestID) > maxRequestIDLength {
|
|
return result, fmt.Errorf("short recommend request ID exceeds %d bytes", maxRequestIDLength)
|
|
}
|
|
opts.maxBatches = clamp(opts.maxBatches, 1, maxAllowedBatches)
|
|
opts.scanMultiplier = clamp(opts.scanMultiplier, 1, maxAllowedMultiplier)
|
|
scanBudget := size * opts.scanMultiplier
|
|
if scanBudget < size {
|
|
scanBudget = size
|
|
}
|
|
if batchBudget := size * opts.maxBatches; scanBudget > batchBudget {
|
|
// 预留不会超过本次最多能查询的范围，避免把永远不会扫描的ID
|
|
// 从Redis传到App进程。
|
|
scanBudget = batchBudget
|
|
}
|
|
|
|
excluded, err := deps.excludedModuleIDs(deps.now(), true)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
excludedSet := make(map[string]struct{}, len(excluded))
|
|
for _, moduleID := range excluded {
|
|
excludedSet[moduleID] = struct{}{}
|
|
}
|
|
|
|
reservation, reserveErr := reserveWithBusyRetry(
|
|
ctx, deps.reserve, uid, scanBudget, requestID,
|
|
)
|
|
result.QueueVersion = reservation.Version
|
|
result.QueueLength = reservation.Length
|
|
if reserveErr != nil {
|
|
return result, reserveErr
|
|
}
|
|
if reservation.Version == "" || reservation.Length <= 0 ||
|
|
reservation.Reserved <= 0 || len(reservation.IDs) == 0 {
|
|
return result, nil
|
|
}
|
|
|
|
committed := false
|
|
defer func() {
|
|
if committed {
|
|
return
|
|
}
|
|
abortCtx, cancel := context.WithTimeout(context.Background(), reservationAbortTTL)
|
|
defer cancel()
|
|
_ = deps.abort(abortCtx, uid, reservation)
|
|
}()
|
|
|
|
result.Videos = make([]*vidmod.VideoModel, 0, size)
|
|
seen := make(map[primitive.ObjectID]struct{}, size)
|
|
cursor := 0
|
|
for cursor < len(reservation.IDs) &&
|
|
(reservation.AlreadyCommitted ||
|
|
(result.Batches < opts.maxBatches && len(result.Videos) < size)) {
|
|
batchSize := size
|
|
if !reservation.AlreadyCommitted {
|
|
batchSize = size - len(result.Videos)
|
|
} else if batchSize > 0 {
|
|
// 已提交请求的receipt记录的是原实际消费前缀。重试必须扫描并
|
|
// 确认完整前缀，再以原consumed幂等提交，不能因当前状态变化
|
|
// 缩短已确认的消费范围。固定按请求大小分批，避免当前只差一条
|
|
// 有效视频时退化为大量单ID Mongo查询。
|
|
}
|
|
if remaining := len(reservation.IDs) - cursor; batchSize > remaining {
|
|
batchSize = remaining
|
|
}
|
|
if batchSize <= 0 {
|
|
break
|
|
}
|
|
end := cursor + batchSize
|
|
batchIDs := reservation.IDs[cursor:end]
|
|
objectIDs := parseUniqueObjectIDs(batchIDs)
|
|
videos, findErr := deps.findVideos(ctx, objectIDs)
|
|
if findErr != nil {
|
|
result.Videos = nil
|
|
return result, findErr
|
|
}
|
|
result.Batches++
|
|
result.Scanned += len(batchIDs)
|
|
cursor = end
|
|
byID := make(map[primitive.ObjectID]*vidmod.VideoModel, len(videos))
|
|
for _, video := range videos {
|
|
if video == nil {
|
|
continue
|
|
}
|
|
if _, blocked := excludedSet[video.MID]; blocked {
|
|
continue
|
|
}
|
|
byID[video.ID] = video
|
|
}
|
|
before := len(result.Videos)
|
|
for _, rawID := range batchIDs {
|
|
if len(result.Videos) >= size {
|
|
break
|
|
}
|
|
id, parseErr := primitive.ObjectIDFromHex(rawID)
|
|
if parseErr != nil {
|
|
continue
|
|
}
|
|
if _, exists := seen[id]; exists {
|
|
continue
|
|
}
|
|
video := byID[id]
|
|
if video == nil {
|
|
continue
|
|
}
|
|
seen[id] = struct{}{}
|
|
result.Videos = append(result.Videos, video)
|
|
if len(result.Videos) == size {
|
|
break
|
|
}
|
|
}
|
|
result.Filtered += len(batchIDs) - (len(result.Videos) - before)
|
|
}
|
|
if result.Scanned <= 0 {
|
|
result.Videos = nil
|
|
return result, fmt.Errorf("short recommend reservation was not scanned")
|
|
}
|
|
commitErr := deps.commit(ctx, uid, reservation, result.Scanned)
|
|
if commitErr != nil && ctx.Err() == nil {
|
|
// 同一receipt重试可确认“脚本已执行但响应丢失”的不确定提交，
|
|
// 不会再次推进offset。
|
|
commitErr = deps.commit(ctx, uid, reservation, result.Scanned)
|
|
}
|
|
if commitErr != nil {
|
|
result.Videos = nil
|
|
return result, commitErr
|
|
}
|
|
committed = true
|
|
result.BudgetExceeded = len(result.Videos) < size &&
|
|
result.QueueLength > 0 &&
|
|
result.Scanned < result.QueueLength &&
|
|
(result.Scanned >= scanBudget || result.Batches >= opts.maxBatches)
|
|
return result, nil
|
|
}
|
|
|
|
func reserveWithBusyRetry(
|
|
ctx context.Context,
|
|
reserve reserveFunc,
|
|
uid uint64,
|
|
size int,
|
|
requestID string,
|
|
) (recommendqueue.Reservation, error) {
|
|
var reservation recommendqueue.Reservation
|
|
var err error
|
|
for attempt := 0; attempt <= reservationBusyRetries; attempt++ {
|
|
reservation, err = reserve(ctx, uid, size, requestID)
|
|
if !errors.Is(err, recommendqueue.ErrReservationBusy) ||
|
|
attempt == reservationBusyRetries {
|
|
return reservation, err
|
|
}
|
|
delay := reservationBusyBackoff << attempt
|
|
timer := time.NewTimer(delay)
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
return reservation, ctx.Err()
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
return reservation, err
|
|
}
|
|
|
|
func parseUniqueObjectIDs(ids []string) []primitive.ObjectID {
|
|
out := make([]primitive.ObjectID, 0, len(ids))
|
|
seen := make(map[primitive.ObjectID]struct{}, len(ids))
|
|
for _, rawID := range ids {
|
|
id, err := primitive.ObjectIDFromHex(rawID)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if _, exists := seen[id]; exists {
|
|
continue
|
|
}
|
|
seen[id] = struct{}{}
|
|
out = append(out, id)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func clamp(value, low, high int) int {
|
|
if value < low {
|
|
return low
|
|
}
|
|
if value > high {
|
|
return high
|
|
}
|
|
return value
|
|
}
|