Claude Opus 5
@@ -0,0 +1,523 @@
|
||||
package shortrecommendser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
recommendqueue "91porn-server/common/shortrecommend"
|
||||
"91porn-server/models/v/vidmod"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
)
|
||||
|
||||
func TestFetchReservesOncePreservesOrderAndCommitsConsumedPrefix(t *testing.T) {
|
||||
validA := primitive.NewObjectID()
|
||||
downShelf := primitive.NewObjectID()
|
||||
excluded := primitive.NewObjectID()
|
||||
validB := primitive.NewObjectID()
|
||||
ids := []string{validA.Hex(), downShelf.Hex(), excluded.Hex(), validB.Hex()}
|
||||
deps := testFetchDependencies(
|
||||
testReservation(ids, len(ids)),
|
||||
func(_ context.Context, batch []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
|
||||
out := make([]*vidmod.VideoModel, 0, len(batch))
|
||||
for _, id := range batch {
|
||||
switch id {
|
||||
case validA, validB:
|
||||
out = append(out, &vidmod.VideoModel{ID: id})
|
||||
case excluded:
|
||||
out = append(out, &vidmod.VideoModel{ID: id, MID: "blocked"})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
)
|
||||
deps.excludedModuleIDs = func(time.Time, bool) ([]string, error) {
|
||||
return []string{"blocked"}, nil
|
||||
}
|
||||
reserveCalls, committed, aborted := 0, 0, 0
|
||||
originalReserve := deps.reserve
|
||||
deps.reserve = func(
|
||||
ctx context.Context, uid uint64, size int, requestID string,
|
||||
) (recommendqueue.Reservation, error) {
|
||||
reserveCalls++
|
||||
if size != 10 {
|
||||
t.Fatalf("reserve size = %d, want scan budget 10", size)
|
||||
}
|
||||
return originalReserve(ctx, uid, size, requestID)
|
||||
}
|
||||
deps.commit = func(
|
||||
_ context.Context,
|
||||
_ uint64,
|
||||
_ recommendqueue.Reservation,
|
||||
consumed int,
|
||||
) error {
|
||||
committed = consumed
|
||||
return nil
|
||||
}
|
||||
deps.abort = func(
|
||||
context.Context, uint64, recommendqueue.Reservation,
|
||||
) error {
|
||||
aborted++
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := fetch(context.Background(), 7, 2, "request-1", fetchOptions{
|
||||
maxBatches: 5, scanMultiplier: 5,
|
||||
}, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch() error = %v", err)
|
||||
}
|
||||
if len(result.Videos) != 2 ||
|
||||
result.Videos[0].ID != validA ||
|
||||
result.Videos[1].ID != validB {
|
||||
t.Fatalf("videos = %#v, want queue ordered valid videos", result.Videos)
|
||||
}
|
||||
if reserveCalls != 1 || committed != 4 || aborted != 0 {
|
||||
t.Fatalf("reserve=%d committed=%d aborted=%d", reserveCalls, committed, aborted)
|
||||
}
|
||||
if result.Scanned != 4 || result.Filtered != 2 || result.Batches != 3 {
|
||||
t.Fatalf("metrics = %+v", result)
|
||||
}
|
||||
if result.QueueVersion != "20260731-r1" || result.QueueLength != len(ids) {
|
||||
t.Fatalf("queue metadata = %+v", result)
|
||||
}
|
||||
if result.BudgetExceeded {
|
||||
t.Fatal("BudgetExceeded = true, want false after a full queue scan")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchCommitsOnlyBoundedScannedPrefix(t *testing.T) {
|
||||
ids := make([]string, 30)
|
||||
for i := range ids {
|
||||
ids[i] = primitive.NewObjectID().Hex()
|
||||
}
|
||||
reservation := testReservation(ids, 100)
|
||||
deps := testFetchDependencies(
|
||||
reservation,
|
||||
func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
|
||||
return nil, nil
|
||||
},
|
||||
)
|
||||
committed := 0
|
||||
deps.commit = func(
|
||||
_ context.Context,
|
||||
_ uint64,
|
||||
_ recommendqueue.Reservation,
|
||||
consumed int,
|
||||
) error {
|
||||
committed = consumed
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := fetch(context.Background(), 8, 3, "request-2", fetchOptions{
|
||||
maxBatches: 2, scanMultiplier: 10,
|
||||
}, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch() error = %v", err)
|
||||
}
|
||||
if result.Batches != 2 || result.Scanned != 6 || committed != 6 {
|
||||
t.Fatalf("committed=%d result=%+v", committed, result)
|
||||
}
|
||||
if !result.BudgetExceeded {
|
||||
t.Fatal("BudgetExceeded = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchMongoFailureAbortsWithoutCommitOrPartialResponse(t *testing.T) {
|
||||
valid := primitive.NewObjectID()
|
||||
filtered := primitive.NewObjectID()
|
||||
failing := primitive.NewObjectID()
|
||||
wantErr := errors.New("mongo unavailable")
|
||||
findCalls := 0
|
||||
deps := testFetchDependencies(
|
||||
testReservation(
|
||||
[]string{valid.Hex(), filtered.Hex(), failing.Hex()},
|
||||
10,
|
||||
),
|
||||
func(_ context.Context, ids []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
|
||||
findCalls++
|
||||
if findCalls == 1 {
|
||||
return []*vidmod.VideoModel{{ID: valid}}, nil
|
||||
}
|
||||
return nil, wantErr
|
||||
},
|
||||
)
|
||||
commitCalls, abortCalls := 0, 0
|
||||
deps.commit = func(
|
||||
context.Context, uint64, recommendqueue.Reservation, int,
|
||||
) error {
|
||||
commitCalls++
|
||||
return nil
|
||||
}
|
||||
deps.abort = func(
|
||||
context.Context, uint64, recommendqueue.Reservation,
|
||||
) error {
|
||||
abortCalls++
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := fetch(context.Background(), 9, 2, "request-3", fetchOptions{
|
||||
maxBatches: 5, scanMultiplier: 5,
|
||||
}, deps)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("error = %v, want %v", err, wantErr)
|
||||
}
|
||||
if result.Videos != nil || commitCalls != 0 || abortCalls != 1 {
|
||||
t.Fatalf("result=%+v commitCalls=%d abortCalls=%d", result, commitCalls, abortCalls)
|
||||
}
|
||||
if result.Scanned != 2 || result.Batches != 1 {
|
||||
t.Fatalf("successful scan metrics = %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRetriesUncertainCommitWithSameReservation(t *testing.T) {
|
||||
id := primitive.NewObjectID()
|
||||
wantErr := errors.New("connection reset after write")
|
||||
deps := testFetchDependencies(
|
||||
testReservation([]string{id.Hex()}, 10),
|
||||
func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
|
||||
return []*vidmod.VideoModel{{ID: id}}, nil
|
||||
},
|
||||
)
|
||||
commitCalls := 0
|
||||
deps.commit = func(
|
||||
context.Context, uint64, recommendqueue.Reservation, int,
|
||||
) error {
|
||||
commitCalls++
|
||||
if commitCalls == 1 {
|
||||
return wantErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := fetch(context.Background(), 10, 1, "request-4", fetchOptions{
|
||||
maxBatches: 5, scanMultiplier: 5,
|
||||
}, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch() error = %v", err)
|
||||
}
|
||||
if commitCalls != 2 || len(result.Videos) != 1 {
|
||||
t.Fatalf("commitCalls=%d result=%+v", commitCalls, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchCommitFailureAbortsAndSuppressesVideos(t *testing.T) {
|
||||
id := primitive.NewObjectID()
|
||||
wantErr := errors.New("redis unavailable")
|
||||
deps := testFetchDependencies(
|
||||
testReservation([]string{id.Hex()}, 10),
|
||||
func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
|
||||
return []*vidmod.VideoModel{{ID: id}}, nil
|
||||
},
|
||||
)
|
||||
commitCalls, abortCalls := 0, 0
|
||||
deps.commit = func(
|
||||
context.Context, uint64, recommendqueue.Reservation, int,
|
||||
) error {
|
||||
commitCalls++
|
||||
return wantErr
|
||||
}
|
||||
deps.abort = func(
|
||||
context.Context, uint64, recommendqueue.Reservation,
|
||||
) error {
|
||||
abortCalls++
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := fetch(context.Background(), 11, 1, "request-5", fetchOptions{
|
||||
maxBatches: 5, scanMultiplier: 5,
|
||||
}, deps)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("error = %v, want %v", err, wantErr)
|
||||
}
|
||||
if commitCalls != 2 || abortCalls != 1 || result.Videos != nil {
|
||||
t.Fatalf("commit=%d abort=%d result=%+v", commitCalls, abortCalls, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchReservationErrorCarriesQueueMetadata(t *testing.T) {
|
||||
deps := testFetchDependencies(recommendqueue.Reservation{}, nil)
|
||||
deps.reserve = func(
|
||||
context.Context, uint64, int, string,
|
||||
) (recommendqueue.Reservation, error) {
|
||||
return recommendqueue.Reservation{
|
||||
Version: "20260731-r2",
|
||||
Length: 100,
|
||||
}, recommendqueue.ErrReservationBusy
|
||||
}
|
||||
|
||||
result, err := fetch(context.Background(), 12, 1, "", fetchOptions{}, deps)
|
||||
if !errors.Is(err, recommendqueue.ErrReservationBusy) {
|
||||
t.Fatalf("error = %v, want ErrReservationBusy", err)
|
||||
}
|
||||
if result.QueueVersion != "20260731-r2" || result.QueueLength != 100 {
|
||||
t.Fatalf("result = %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRetriesBusyReservationWithinRequest(t *testing.T) {
|
||||
id := primitive.NewObjectID()
|
||||
reservation := testReservation([]string{id.Hex()}, 10)
|
||||
deps := testFetchDependencies(
|
||||
reservation,
|
||||
func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
|
||||
return []*vidmod.VideoModel{{ID: id}}, nil
|
||||
},
|
||||
)
|
||||
reserveCalls := 0
|
||||
deps.reserve = func(
|
||||
context.Context, uint64, int, string,
|
||||
) (recommendqueue.Reservation, error) {
|
||||
reserveCalls++
|
||||
if reserveCalls < 3 {
|
||||
return recommendqueue.Reservation{
|
||||
Version: reservation.Version,
|
||||
Length: reservation.Length,
|
||||
}, recommendqueue.ErrReservationBusy
|
||||
}
|
||||
return reservation, nil
|
||||
}
|
||||
|
||||
result, err := fetch(context.Background(), 12, 1, "", fetchOptions{}, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch() error = %v", err)
|
||||
}
|
||||
if reserveCalls != 3 || len(result.Videos) != 1 {
|
||||
t.Fatalf("reserveCalls=%d result=%+v", reserveCalls, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReserveBusyRetryHonorsCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
calls := 0
|
||||
_, err := reserveWithBusyRetry(
|
||||
ctx,
|
||||
func(
|
||||
context.Context, uint64, int, string,
|
||||
) (recommendqueue.Reservation, error) {
|
||||
calls++
|
||||
cancel()
|
||||
return recommendqueue.Reservation{}, recommendqueue.ErrReservationBusy
|
||||
},
|
||||
1,
|
||||
20,
|
||||
"",
|
||||
)
|
||||
if !errors.Is(err, context.Canceled) || calls != 1 {
|
||||
t.Fatalf("error=%v calls=%d", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRejectsCanceledContextBeforeDependencies(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
called := false
|
||||
deps := testFetchDependencies(recommendqueue.Reservation{}, nil)
|
||||
deps.reserve = func(
|
||||
context.Context, uint64, int, string,
|
||||
) (recommendqueue.Reservation, error) {
|
||||
called = true
|
||||
return recommendqueue.Reservation{}, nil
|
||||
}
|
||||
|
||||
_, err := fetch(ctx, 13, 1, "request-6", fetchOptions{}, deps)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("error = %v, want context.Canceled", err)
|
||||
}
|
||||
if called {
|
||||
t.Fatal("dependency called after context cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchWithoutClientRequestIDUsesReservationWithoutBusinessReceiptScope(t *testing.T) {
|
||||
id := primitive.NewObjectID()
|
||||
deps := testFetchDependencies(
|
||||
testReservation([]string{id.Hex()}, 1),
|
||||
func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
|
||||
return []*vidmod.VideoModel{{ID: id}}, nil
|
||||
},
|
||||
)
|
||||
gotRequestID := "not-called"
|
||||
originalReserve := deps.reserve
|
||||
deps.reserve = func(
|
||||
ctx context.Context, uid uint64, size int, requestID string,
|
||||
) (recommendqueue.Reservation, error) {
|
||||
gotRequestID = requestID
|
||||
return originalReserve(ctx, uid, size, requestID)
|
||||
}
|
||||
|
||||
result, err := fetch(context.Background(), 14, 1, "", fetchOptions{
|
||||
maxBatches: 5, scanMultiplier: 5,
|
||||
}, deps)
|
||||
if err != nil {
|
||||
t.Fatalf("fetch() error = %v", err)
|
||||
}
|
||||
if gotRequestID != "" || len(result.Videos) != 1 {
|
||||
t.Fatalf("requestID=%q result=%+v", gotRequestID, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchCommittedReceiptReplaysEntireConsumedPrefix(t *testing.T) {
|
||||
ids := []string{
|
||||
primitive.NewObjectID().Hex(),
|
||||
primitive.NewObjectID().Hex(),
|
||||
primitive.NewObjectID().Hex(),
|
||||
}
|
||||
reservation := testReservation(ids, 10)
|
||||
reservation.AlreadyCommitted = true
|
||||
deps := testFetchDependencies(
|
||||
reservation,
|
||||
func(_ context.Context, batch []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
|
||||
out := make([]*vidmod.VideoModel, 0, len(batch))
|
||||
for _, id := range batch {
|
||||
out = append(out, &vidmod.VideoModel{ID: id})
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
)
|
||||
committed := 0
|
||||
deps.commit = func(
|
||||
_ context.Context,
|
||||
_ uint64,
|
||||
_ recommendqueue.Reservation,
|
||||
consumed int,
|
||||
) error {
|
||||
committed = consumed
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := fetch(context.Background(), 15, 1, "request-retry", fetchOptions{
|
||||
maxBatches: 1, scanMultiplier: 1,
|
||||
}, deps)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if committed != len(ids) || result.Scanned != len(ids) ||
|
||||
len(result.Videos) != 1 {
|
||||
t.Fatalf("committed=%d result=%+v", committed, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchCommittedReceiptUsesBoundedMongoBatches(t *testing.T) {
|
||||
const (
|
||||
requestSize = 20
|
||||
consumed = 100
|
||||
)
|
||||
ids := make([]string, consumed)
|
||||
for i := range ids {
|
||||
ids[i] = primitive.NewObjectID().Hex()
|
||||
}
|
||||
reservation := testReservation(ids, consumed)
|
||||
reservation.AlreadyCommitted = true
|
||||
batchSizes := make([]int, 0, consumed/requestSize)
|
||||
deps := testFetchDependencies(
|
||||
reservation,
|
||||
func(_ context.Context, batch []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
|
||||
batchSizes = append(batchSizes, len(batch))
|
||||
if len(batchSizes) > 1 {
|
||||
return nil, nil
|
||||
}
|
||||
videos := make([]*vidmod.VideoModel, 0, requestSize-1)
|
||||
for _, id := range batch[:requestSize-1] {
|
||||
videos = append(videos, &vidmod.VideoModel{ID: id})
|
||||
}
|
||||
return videos, nil
|
||||
},
|
||||
)
|
||||
committed := 0
|
||||
deps.commit = func(
|
||||
_ context.Context,
|
||||
_ uint64,
|
||||
_ recommendqueue.Reservation,
|
||||
value int,
|
||||
) error {
|
||||
committed = value
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := fetch(context.Background(), 16, requestSize, "request-retry", fetchOptions{
|
||||
maxBatches: 1, scanMultiplier: 1,
|
||||
}, deps)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(batchSizes) != consumed/requestSize {
|
||||
t.Fatalf("Mongo calls=%d batchSizes=%v", len(batchSizes), batchSizes)
|
||||
}
|
||||
for _, batchSize := range batchSizes {
|
||||
if batchSize != requestSize {
|
||||
t.Fatalf("batchSizes=%v", batchSizes)
|
||||
}
|
||||
}
|
||||
if committed != consumed || result.Scanned != consumed ||
|
||||
len(result.Videos) != requestSize-1 {
|
||||
t.Fatalf("committed=%d result=%+v", committed, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedRequestIDSeparatesEntryAndRequestSize(t *testing.T) {
|
||||
first, err := scopedRequestID("recommend-list", 20, "request")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same, _ := scopedRequestID("recommend-list", 20, " request ")
|
||||
otherEntry, _ := scopedRequestID("module-short-all", 20, "request")
|
||||
otherSize, _ := scopedRequestID("recommend-list", 10, "request")
|
||||
if first == "" || first != same {
|
||||
t.Fatalf("scoped IDs first=%q same=%q", first, same)
|
||||
}
|
||||
if first == otherEntry || first == otherSize || otherEntry == otherSize {
|
||||
t.Fatalf("scope collision: %q %q %q", first, otherEntry, otherSize)
|
||||
}
|
||||
blank, err := scopedRequestID("recommend-list", 20, " ")
|
||||
if err != nil || blank != "" {
|
||||
t.Fatalf("blank request ID = %q, %v", blank, err)
|
||||
}
|
||||
}
|
||||
|
||||
func testReservation(ids []string, length int) recommendqueue.Reservation {
|
||||
return recommendqueue.Reservation{
|
||||
Version: "20260731-r1",
|
||||
Length: length,
|
||||
Offset: 0,
|
||||
Reserved: len(ids),
|
||||
IDs: append([]string(nil), ids...),
|
||||
LeaseToken: "lease",
|
||||
ReceiptID: "receipt",
|
||||
}
|
||||
}
|
||||
|
||||
func testFetchDependencies(
|
||||
reservation recommendqueue.Reservation,
|
||||
find func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error),
|
||||
) fetchDependencies {
|
||||
if find == nil {
|
||||
find = func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
return fetchDependencies{
|
||||
reserve: func(
|
||||
context.Context, uint64, int, string,
|
||||
) (recommendqueue.Reservation, error) {
|
||||
return reservation, nil
|
||||
},
|
||||
commit: func(
|
||||
context.Context, uint64, recommendqueue.Reservation, int,
|
||||
) error {
|
||||
return nil
|
||||
},
|
||||
abort: func(
|
||||
context.Context, uint64, recommendqueue.Reservation,
|
||||
) error {
|
||||
return nil
|
||||
},
|
||||
excludedModuleIDs: func(time.Time, bool) ([]string, error) {
|
||||
return nil, nil
|
||||
},
|
||||
findVideos: find,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user