huangguo_server/app/service/shortrecommendser/fetch_test.go

package shortrecommendser

import (
	"context"
	"errors"
	"testing"
	"time"

	recommendqueue "91porn-server/common/shortrecommend"
	"91porn-server/models/v/vidmod"

	"go.mongodb.org/mongo-driver/bson/primitive"
)

func TestFetchReservesOncePreservesOrderAndCommitsConsumedPrefix(t *testing.T) {
	validA := primitive.NewObjectID()
	downShelf := primitive.NewObjectID()
	excluded := primitive.NewObjectID()
	validB := primitive.NewObjectID()
	ids := []string{validA.Hex(), downShelf.Hex(), excluded.Hex(), validB.Hex()}
	deps := testFetchDependencies(
		testReservation(ids, len(ids)),
		func(_ context.Context, batch []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
			out := make([]*vidmod.VideoModel, 0, len(batch))
			for _, id := range batch {
				switch id {
				case validA, validB:
					out = append(out, &vidmod.VideoModel{ID: id})
				case excluded:
					out = append(out, &vidmod.VideoModel{ID: id, MID: "blocked"})
				}
			}
			return out, nil
		},
	)
	deps.excludedModuleIDs = func(time.Time, bool) ([]string, error) {
		return []string{"blocked"}, nil
	}
	reserveCalls, committed, aborted := 0, 0, 0
	originalReserve := deps.reserve
	deps.reserve = func(
		ctx context.Context, uid uint64, size int, requestID string,
	) (recommendqueue.Reservation, error) {
		reserveCalls++
		if size != 10 {
			t.Fatalf("reserve size = %d, want scan budget 10", size)
		}
		return originalReserve(ctx, uid, size, requestID)
	}
	deps.commit = func(
		_ context.Context,
		_ uint64,
		_ recommendqueue.Reservation,
		consumed int,
	) error {
		committed = consumed
		return nil
	}
	deps.abort = func(
		context.Context, uint64, recommendqueue.Reservation,
	) error {
		aborted++
		return nil
	}

	result, err := fetch(context.Background(), 7, 2, "request-1", fetchOptions{
		maxBatches: 5, scanMultiplier: 5,
	}, deps)
	if err != nil {
		t.Fatalf("fetch() error = %v", err)
	}
	if len(result.Videos) != 2 ||
		result.Videos[0].ID != validA ||
		result.Videos[1].ID != validB {
		t.Fatalf("videos = %#v, want queue ordered valid videos", result.Videos)
	}
	if reserveCalls != 1 || committed != 4 || aborted != 0 {
		t.Fatalf("reserve=%d committed=%d aborted=%d", reserveCalls, committed, aborted)
	}
	if result.Scanned != 4 || result.Filtered != 2 || result.Batches != 3 {
		t.Fatalf("metrics = %+v", result)
	}
	if result.QueueVersion != "20260731-r1" || result.QueueLength != len(ids) {
		t.Fatalf("queue metadata = %+v", result)
	}
	if result.BudgetExceeded {
		t.Fatal("BudgetExceeded = true, want false after a full queue scan")
	}
}

func TestFetchCommitsOnlyBoundedScannedPrefix(t *testing.T) {
	ids := make([]string, 30)
	for i := range ids {
		ids[i] = primitive.NewObjectID().Hex()
	}
	reservation := testReservation(ids, 100)
	deps := testFetchDependencies(
		reservation,
		func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
			return nil, nil
		},
	)
	committed := 0
	deps.commit = func(
		_ context.Context,
		_ uint64,
		_ recommendqueue.Reservation,
		consumed int,
	) error {
		committed = consumed
		return nil
	}

	result, err := fetch(context.Background(), 8, 3, "request-2", fetchOptions{
		maxBatches: 2, scanMultiplier: 10,
	}, deps)
	if err != nil {
		t.Fatalf("fetch() error = %v", err)
	}
	if result.Batches != 2 || result.Scanned != 6 || committed != 6 {
		t.Fatalf("committed=%d result=%+v", committed, result)
	}
	if !result.BudgetExceeded {
		t.Fatal("BudgetExceeded = false, want true")
	}
}

func TestFetchMongoFailureAbortsWithoutCommitOrPartialResponse(t *testing.T) {
	valid := primitive.NewObjectID()
	filtered := primitive.NewObjectID()
	failing := primitive.NewObjectID()
	wantErr := errors.New("mongo unavailable")
	findCalls := 0
	deps := testFetchDependencies(
		testReservation(
			[]string{valid.Hex(), filtered.Hex(), failing.Hex()},
			10,
		),
		func(_ context.Context, ids []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
			findCalls++
			if findCalls == 1 {
				return []*vidmod.VideoModel{{ID: valid}}, nil
			}
			return nil, wantErr
		},
	)
	commitCalls, abortCalls := 0, 0
	deps.commit = func(
		context.Context, uint64, recommendqueue.Reservation, int,
	) error {
		commitCalls++
		return nil
	}
	deps.abort = func(
		context.Context, uint64, recommendqueue.Reservation,
	) error {
		abortCalls++
		return nil
	}

	result, err := fetch(context.Background(), 9, 2, "request-3", fetchOptions{
		maxBatches: 5, scanMultiplier: 5,
	}, deps)
	if !errors.Is(err, wantErr) {
		t.Fatalf("error = %v, want %v", err, wantErr)
	}
	if result.Videos != nil || commitCalls != 0 || abortCalls != 1 {
		t.Fatalf("result=%+v commitCalls=%d abortCalls=%d", result, commitCalls, abortCalls)
	}
	if result.Scanned != 2 || result.Batches != 1 {
		t.Fatalf("successful scan metrics = %+v", result)
	}
}

func TestFetchRetriesUncertainCommitWithSameReservation(t *testing.T) {
	id := primitive.NewObjectID()
	wantErr := errors.New("connection reset after write")
	deps := testFetchDependencies(
		testReservation([]string{id.Hex()}, 10),
		func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
			return []*vidmod.VideoModel{{ID: id}}, nil
		},
	)
	commitCalls := 0
	deps.commit = func(
		context.Context, uint64, recommendqueue.Reservation, int,
	) error {
		commitCalls++
		if commitCalls == 1 {
			return wantErr
		}
		return nil
	}

	result, err := fetch(context.Background(), 10, 1, "request-4", fetchOptions{
		maxBatches: 5, scanMultiplier: 5,
	}, deps)
	if err != nil {
		t.Fatalf("fetch() error = %v", err)
	}
	if commitCalls != 2 || len(result.Videos) != 1 {
		t.Fatalf("commitCalls=%d result=%+v", commitCalls, result)
	}
}

func TestFetchCommitFailureAbortsAndSuppressesVideos(t *testing.T) {
	id := primitive.NewObjectID()
	wantErr := errors.New("redis unavailable")
	deps := testFetchDependencies(
		testReservation([]string{id.Hex()}, 10),
		func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
			return []*vidmod.VideoModel{{ID: id}}, nil
		},
	)
	commitCalls, abortCalls := 0, 0
	deps.commit = func(
		context.Context, uint64, recommendqueue.Reservation, int,
	) error {
		commitCalls++
		return wantErr
	}
	deps.abort = func(
		context.Context, uint64, recommendqueue.Reservation,
	) error {
		abortCalls++
		return nil
	}

	result, err := fetch(context.Background(), 11, 1, "request-5", fetchOptions{
		maxBatches: 5, scanMultiplier: 5,
	}, deps)
	if !errors.Is(err, wantErr) {
		t.Fatalf("error = %v, want %v", err, wantErr)
	}
	if commitCalls != 2 || abortCalls != 1 || result.Videos != nil {
		t.Fatalf("commit=%d abort=%d result=%+v", commitCalls, abortCalls, result)
	}
}

func TestFetchReservationErrorCarriesQueueMetadata(t *testing.T) {
	deps := testFetchDependencies(recommendqueue.Reservation{}, nil)
	deps.reserve = func(
		context.Context, uint64, int, string,
	) (recommendqueue.Reservation, error) {
		return recommendqueue.Reservation{
			Version: "20260731-r2",
			Length:  100,
		}, recommendqueue.ErrReservationBusy
	}

	result, err := fetch(context.Background(), 12, 1, "", fetchOptions{}, deps)
	if !errors.Is(err, recommendqueue.ErrReservationBusy) {
		t.Fatalf("error = %v, want ErrReservationBusy", err)
	}
	if result.QueueVersion != "20260731-r2" || result.QueueLength != 100 {
		t.Fatalf("result = %+v", result)
	}
}

func TestFetchRetriesBusyReservationWithinRequest(t *testing.T) {
	id := primitive.NewObjectID()
	reservation := testReservation([]string{id.Hex()}, 10)
	deps := testFetchDependencies(
		reservation,
		func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
			return []*vidmod.VideoModel{{ID: id}}, nil
		},
	)
	reserveCalls := 0
	deps.reserve = func(
		context.Context, uint64, int, string,
	) (recommendqueue.Reservation, error) {
		reserveCalls++
		if reserveCalls < 3 {
			return recommendqueue.Reservation{
				Version: reservation.Version,
				Length:  reservation.Length,
			}, recommendqueue.ErrReservationBusy
		}
		return reservation, nil
	}

	result, err := fetch(context.Background(), 12, 1, "", fetchOptions{}, deps)
	if err != nil {
		t.Fatalf("fetch() error = %v", err)
	}
	if reserveCalls != 3 || len(result.Videos) != 1 {
		t.Fatalf("reserveCalls=%d result=%+v", reserveCalls, result)
	}
}

func TestReserveBusyRetryHonorsCancellation(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	calls := 0
	_, err := reserveWithBusyRetry(
		ctx,
		func(
			context.Context, uint64, int, string,
		) (recommendqueue.Reservation, error) {
			calls++
			cancel()
			return recommendqueue.Reservation{}, recommendqueue.ErrReservationBusy
		},
		1,
		20,
		"",
	)
	if !errors.Is(err, context.Canceled) || calls != 1 {
		t.Fatalf("error=%v calls=%d", err, calls)
	}
}

func TestFetchRejectsCanceledContextBeforeDependencies(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	called := false
	deps := testFetchDependencies(recommendqueue.Reservation{}, nil)
	deps.reserve = func(
		context.Context, uint64, int, string,
	) (recommendqueue.Reservation, error) {
		called = true
		return recommendqueue.Reservation{}, nil
	}

	_, err := fetch(ctx, 13, 1, "request-6", fetchOptions{}, deps)
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("error = %v, want context.Canceled", err)
	}
	if called {
		t.Fatal("dependency called after context cancellation")
	}
}

func TestFetchWithoutClientRequestIDUsesReservationWithoutBusinessReceiptScope(t *testing.T) {
	id := primitive.NewObjectID()
	deps := testFetchDependencies(
		testReservation([]string{id.Hex()}, 1),
		func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
			return []*vidmod.VideoModel{{ID: id}}, nil
		},
	)
	gotRequestID := "not-called"
	originalReserve := deps.reserve
	deps.reserve = func(
		ctx context.Context, uid uint64, size int, requestID string,
	) (recommendqueue.Reservation, error) {
		gotRequestID = requestID
		return originalReserve(ctx, uid, size, requestID)
	}

	result, err := fetch(context.Background(), 14, 1, "", fetchOptions{
		maxBatches: 5, scanMultiplier: 5,
	}, deps)
	if err != nil {
		t.Fatalf("fetch() error = %v", err)
	}
	if gotRequestID != "" || len(result.Videos) != 1 {
		t.Fatalf("requestID=%q result=%+v", gotRequestID, result)
	}
}

func TestFetchCommittedReceiptReplaysEntireConsumedPrefix(t *testing.T) {
	ids := []string{
		primitive.NewObjectID().Hex(),
		primitive.NewObjectID().Hex(),
		primitive.NewObjectID().Hex(),
	}
	reservation := testReservation(ids, 10)
	reservation.AlreadyCommitted = true
	deps := testFetchDependencies(
		reservation,
		func(_ context.Context, batch []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
			out := make([]*vidmod.VideoModel, 0, len(batch))
			for _, id := range batch {
				out = append(out, &vidmod.VideoModel{ID: id})
			}
			return out, nil
		},
	)
	committed := 0
	deps.commit = func(
		_ context.Context,
		_ uint64,
		_ recommendqueue.Reservation,
		consumed int,
	) error {
		committed = consumed
		return nil
	}

	result, err := fetch(context.Background(), 15, 1, "request-retry", fetchOptions{
		maxBatches: 1, scanMultiplier: 1,
	}, deps)
	if err != nil {
		t.Fatal(err)
	}
	if committed != len(ids) || result.Scanned != len(ids) ||
		len(result.Videos) != 1 {
		t.Fatalf("committed=%d result=%+v", committed, result)
	}
}

func TestFetchCommittedReceiptUsesBoundedMongoBatches(t *testing.T) {
	const (
		requestSize = 20
		consumed    = 100
	)
	ids := make([]string, consumed)
	for i := range ids {
		ids[i] = primitive.NewObjectID().Hex()
	}
	reservation := testReservation(ids, consumed)
	reservation.AlreadyCommitted = true
	batchSizes := make([]int, 0, consumed/requestSize)
	deps := testFetchDependencies(
		reservation,
		func(_ context.Context, batch []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
			batchSizes = append(batchSizes, len(batch))
			if len(batchSizes) > 1 {
				return nil, nil
			}
			videos := make([]*vidmod.VideoModel, 0, requestSize-1)
			for _, id := range batch[:requestSize-1] {
				videos = append(videos, &vidmod.VideoModel{ID: id})
			}
			return videos, nil
		},
	)
	committed := 0
	deps.commit = func(
		_ context.Context,
		_ uint64,
		_ recommendqueue.Reservation,
		value int,
	) error {
		committed = value
		return nil
	}

	result, err := fetch(context.Background(), 16, requestSize, "request-retry", fetchOptions{
		maxBatches: 1, scanMultiplier: 1,
	}, deps)
	if err != nil {
		t.Fatal(err)
	}
	if len(batchSizes) != consumed/requestSize {
		t.Fatalf("Mongo calls=%d batchSizes=%v", len(batchSizes), batchSizes)
	}
	for _, batchSize := range batchSizes {
		if batchSize != requestSize {
			t.Fatalf("batchSizes=%v", batchSizes)
		}
	}
	if committed != consumed || result.Scanned != consumed ||
		len(result.Videos) != requestSize-1 {
		t.Fatalf("committed=%d result=%+v", committed, result)
	}
}

func TestScopedRequestIDSeparatesEntryAndRequestSize(t *testing.T) {
	first, err := scopedRequestID("recommend-list", 20, "request")
	if err != nil {
		t.Fatal(err)
	}
	same, _ := scopedRequestID("recommend-list", 20, " request ")
	otherEntry, _ := scopedRequestID("module-short-all", 20, "request")
	otherSize, _ := scopedRequestID("recommend-list", 10, "request")
	if first == "" || first != same {
		t.Fatalf("scoped IDs first=%q same=%q", first, same)
	}
	if first == otherEntry || first == otherSize || otherEntry == otherSize {
		t.Fatalf("scope collision: %q %q %q", first, otherEntry, otherSize)
	}
	blank, err := scopedRequestID("recommend-list", 20, " ")
	if err != nil || blank != "" {
		t.Fatalf("blank request ID = %q, %v", blank, err)
	}
}

func testReservation(ids []string, length int) recommendqueue.Reservation {
	return recommendqueue.Reservation{
		Version:    "20260731-r1",
		Length:     length,
		Offset:     0,
		Reserved:   len(ids),
		IDs:        append([]string(nil), ids...),
		LeaseToken: "lease",
		ReceiptID:  "receipt",
	}
}

func testFetchDependencies(
	reservation recommendqueue.Reservation,
	find func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error),
) fetchDependencies {
	if find == nil {
		find = func(context.Context, []primitive.ObjectID) ([]*vidmod.VideoModel, error) {
			return nil, nil
		}
	}
	return fetchDependencies{
		reserve: func(
			context.Context, uint64, int, string,
		) (recommendqueue.Reservation, error) {
			return reservation, nil
		},
		commit: func(
			context.Context, uint64, recommendqueue.Reservation, int,
		) error {
			return nil
		},
		abort: func(
			context.Context, uint64, recommendqueue.Reservation,
		) error {
			return nil
		},
		excludedModuleIDs: func(time.Time, bool) ([]string, error) {
			return nil, nil
		},
		findVideos: find,
		now:        time.Now,
	}
}