Initial commit

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

skd/job/cache/rankcache/run.go

@@ -0,0 +1,51 @@
+package rankcache
+
+import (
+	"91porn-server/app/service/rankser"
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/log"
+	"91porn-server/models/v/usermod"
+	"91porn-server/skd/skdg"
+)
+
+type UserRank = rankser.UserRank
+
+type User = rankser.User
+
+type UBaseInfo = usermod.BaseInfo
+
+const (
+	UserInviteRankType = rankser.UserInviteRankType //推广达人
+	UserIncomeRankType = rankser.UserIncomeRankType //收益达人
+	UserUploadRankType = rankser.UserUploadRankType //上传达人
+)
+
+func Update() {
+	saveUserIncomeRank()
+	saveUserUploadRank()
+}
+
+func saveUserIncomeRank() {
+	rank, err := rankser.GetUserIncomeRank()
+	if err != nil {
+		log.Error("rankcach GetUserIncomeRank error", log.E(err))
+		return
+	}
+	redisKey := redisconst.RankKey(UserIncomeRankType)
+	if err = skdg.Redis.Set(redisKey, rank, redisconst.RankExpireMax); err != nil {
+		log.Warn("rankcach UserIncomeRankType redis set error", log.E(err))
+	}
+}
+
+func saveUserUploadRank() {
+	rank, err := rankser.GetUserUploadRank()
+	if err != nil {
+		log.Error("rankcach GetUserUploadRank error", log.E(err))
+		return
+	}
+	redisKey := redisconst.RankKey(UserUploadRankType)
+	_ = skdg.Redis.Set(redisKey, rank, redisconst.RankExpireMax)
+	if err != nil {
+		log.Warn("rankcach UserUploadRankType redis set error", log.E(err))
+	}
+}

skd/job/cache/tonecache/run.go

@@ -0,0 +1,51 @@
+package tonecache
+
+import (
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/log"
+	"91porn-server/models/v/tonemod"
+	"91porn-server/models/v/vidmod"
+	"91porn-server/skd/skdg"
+)
+
+const MaxVideoCount = 500
+
+func Update() {
+	officialRecomPlayCount, err := vidmod.SumFakePlayCountSortByToneOfficialRecom(MaxVideoCount)
+	if err != nil {
+		log.Error("tonecach OfficialRecomPlayCount SumFakePlayCountSortByToneOfficialRecom faild", log.E(err))
+	}
+	latestUploadPlayCount, err := vidmod.SumFakePlayCountSortByCreatedAt(MaxVideoCount)
+	if err != nil {
+		log.Error("tonecach LatestUploadPlayCount SumFakePlayCountSortByCreatedAt faild", log.E(err))
+	}
+	mostLikesPlayCount, err := vidmod.SumFakePlayCountSortByFakeLikeCount(MaxVideoCount)
+	if err != nil {
+		log.Error("tonecach MostLikesPlayCount SumFakePlayCountSortByFakeLikeCount faild", log.E(err))
+	}
+	playAtMostPlayCount, err := vidmod.SumFakePlayCountSortByFakePlayCount(MaxVideoCount)
+	if err != nil {
+		log.Error("tonecach PlayAtMostPlayCount SumFakePlayCountSortByFakePlayCount faild", log.E(err))
+	}
+	mostCommentsPlayCount, err := vidmod.SumFakePlayCountSortByFakeCommentCount(MaxVideoCount)
+	if err != nil {
+		log.Error("tonecach MostCommentsPlayCount SumFakePlayCountSortByFakeCommentCount faild", log.E(err))
+	}
+	freeAreaPlayCount, err := vidmod.SumFakePlayCountSortByFreeArea(MaxVideoCount)
+	if err != nil {
+		log.Error("tonecach MostCommentsPlayCount SumFakePlayCountSortByFakeCommentCount faild", log.E(err))
+	}
+	toneValue := tonemod.ToneValue{
+		OfficialRecomPlayCount: officialRecomPlayCount,
+		GoldCoinAreaPayCount:   0,
+		LatestUploadPlayCount:  latestUploadPlayCount,
+		MostLikesPlayCount:     mostLikesPlayCount,
+		PlayAtMostPlayCount:    playAtMostPlayCount,
+		MostCommentsPlayCount:  mostCommentsPlayCount,
+		FreeAreaPlayCount:      freeAreaPlayCount,
+	}
+	redisKey := redisconst.ToneKey()
+	if err = skdg.Redis.Set(redisKey, toneValue, redisconst.ToneExpireMax); err != nil {
+		log.Warn("tonecach Update redis set error", log.E(err))
+	}
+}

skd/job/check/daichongcallback.go

@@ -0,0 +1,149 @@
+package check
+
+import (
+	"context"
+	"time"
+
+	"91porn-server/common/daichong"
+	"91porn-server/common/log"
+	"91porn-server/common/pageopt"
+	"91porn-server/common/stderr"
+	"91porn-server/models/commod"
+	"91porn-server/models/v/rchgordmod"
+	"91porn-server/skd/skdg"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+var pageSize int64 = 20
+
+// 通知第三方代充订单结果
+func DaiChongOrderCallBack() {
+	log.Info("DaiChongOrderCallBack start")
+	sort := rchgordmod.Sort{{Key: "createdAt", Value: 1}}
+	daichongType := string(rchgordmod.Rech_DaiChong)
+	matList := []pageopt.Matcher{
+		(&rchgordmod.RechargeTypeMatch{RechargeType: &daichongType}).New(),
+		(&rchgordmod.NotifyStatusMatch{NotifyStatus: []int{rchgordmod.Notify_DEFAULT, rchgordmod.Notify_FAILURE}}).New(),
+		(&rchgordmod.StatusInMatch{Status: []int{rchgordmod.SUCCESS, rchgordmod.FAILURE}}).New(),
+	}
+	var pageNumber int64 = 1 //从第一页开始
+	for {
+		var skip = (pageNumber - 1) * pageSize
+		//订单列表
+		rchgorderList, err := rchgordmod.List(sort, &skip, &pageSize, matList...)
+		if err != nil {
+			return
+		}
+		if len(rchgorderList) == 0 {
+			log.Info("DaiChongOrderCallBack end")
+			return
+		}
+		for _, v := range rchgorderList {
+			if oid, err := primitive.ObjectIDFromHex(v.OID); err == nil {
+				if !oid.IsZero() {
+					continue
+				}
+			}
+			handOrderBack(v)
+		}
+		pageNumber++
+	}
+}
+
+// 将超时支付的订单设置为失败订单,并通知给对方
+func HandleProcessingOrder() {
+	log.Info("HandleProcessingOrder start")
+	//创建时间小于三天前的订单
+	threeDayBefore := time.Now().AddDate(0, 0, -3)
+	processing := rchgordmod.Processing
+	sort := rchgordmod.Sort{{Key: "createdAt", Value: 1}}
+	daichongType := string(rchgordmod.Rech_DaiChong)
+	matList := []pageopt.Matcher{
+		(&rchgordmod.RechargeTypeMatch{RechargeType: &daichongType}).New(),
+		(&rchgordmod.CreatedAtLTMatch{LT: &threeDayBefore}).New(),
+		(&rchgordmod.StatusMatch{Status: &processing}).New(),
+	}
+	var pageNumber int64 = 1 //从第一页开始
+	for {
+		var skip = (pageNumber - 1) * pageSize
+		//订单列表
+		rchgorderList, err := rchgordmod.List(sort, &skip, &pageSize, matList...)
+		if err != nil {
+			return
+		}
+		if len(rchgorderList) == 0 {
+			log.Info("HandleProcessingOrder end")
+			return
+		}
+		for _, v := range rchgorderList {
+			v.Status = rchgordmod.FAILURE
+			v.StatusDesc = rchgordmod.Status(rchgordmod.FAILURE).Desc()
+			v.Remark = "该订单超时三天未处理,失败"
+			v.FailureAt = time.Now()
+			handOrderBack(v)
+		}
+		pageNumber++
+	}
+}
+
+func handOrderBack(v *rchgordmod.RechargeOrder) {
+	var orderStatus int
+	switch v.Status {
+	case rchgordmod.SUCCESS:
+		orderStatus = daichong.Success
+	case rchgordmod.FAILURE:
+		orderStatus = daichong.Failed
+	}
+	if orderStatus == 0 {
+		return
+	}
+	var timestamp = v.NotifyTime
+	if v.NotifyTime == 0 {
+		timestamp = time.Now().UTC().Unix()
+	}
+	resp, code := orderCallBack(daichong.OrderCallSign{
+		OrderId: v.OID,
+		Time:    timestamp,
+		Status:  orderStatus,
+	})
+	selector := rchgordmod.ReplaceEditSelector{
+		Status:     &v.Status,
+		StatusDesc: &v.StatusDesc,
+		Remark:     &v.Remark,
+		FailureAt:  &v.FailureAt,
+		SuccessAt:  &v.SuccessAt,
+	}
+	if code == stderr.Success {
+		notifySuccess := rchgordmod.Notify_SUCCESS
+		selector.NotifyStatus = &notifySuccess
+		log.Info("daichong order call back send http request successfully",
+			log.Any("orderStatus", orderStatus),
+			log.Any("orderId", v.OID),
+			log.Any("time", time.Now().Format(time.RFC3339)))
+	} else {
+		notifyFailure := rchgordmod.FAILURE
+		selector.NotifyStatus = &notifyFailure
+		log.Warn("daichong order call back send http request failed", log.Any("responseCode", code), log.Any("msg", resp.Msg))
+	}
+	_ = rchgordmod.UpdateBaseOnOid(nil, v.OID, selector)
+}
+
+// 上分通知 将订单的处理结果通知给代充平台
+func orderCallBack(req daichong.OrderCallSign) (commod.Resp, stderr.Code) {
+	sign, err := daichong.Convert2Sign(req, skdg.Conf.DaiChong.AppSecret)
+	if err != nil {
+		log.Warn("create sign data wrong", log.Any("warn", err))
+		return commod.Resp{}, stderr.Failure
+	}
+	orderReq := daichong.CommonReq{
+		AppId: skdg.Conf.DaiChong.AppID,
+		Data:  sign,
+	}
+	resp, err := daichong.OrderCallBack(context.TODO(), orderReq, skdg.Conf.DaiChong.Domain)
+	if err != nil {
+		log.Warn("GetOrderInfo send http wrong", log.Any("warn", err))
+		return commod.Resp{}, stderr.Failure
+	}
+	return resp, resp.Code
+}

skd/job/checkin/checkin.go

@@ -0,0 +1,70 @@
+package checkin
+
+import (
+	"91porn-server/common/log"
+	"91porn-server/models/v/checkinconfigmod"
+	"91porn-server/models/v/vidmod"
+	"91porn-server/skd/service"
+
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+var videoCheckRunning bool
+
+// VideoCheckRS 定期检查签到奖励视频的切片状态
+func VideoCheckRS() {
+	if videoCheckRunning {
+		return
+	}
+	videoCheckRunning = true
+	defer func() {
+		videoCheckRunning = false
+	}()
+
+	cfg, err := checkinconfigmod.FindOne(bson.M{})
+	if err != nil || cfg == nil {
+		return
+	}
+	if len(cfg.RewardBgVideos) == 0 {
+		return
+	}
+
+	updated := false
+	for i, v := range cfg.RewardBgVideos {
+		if v.FsResourceId == "" {
+			continue
+		}
+		// 已完成的跳过
+		if v.FsResourceStatus == vidmod.Completed {
+			continue
+		}
+		fileInfo, err := service.PullFileInfo(v.FsResourceId)
+		if err != nil {
+			log.Error("checkin VideoCheckRS PullFileInfo fail",
+				log.Any("fsResourceId", v.FsResourceId), log.E(err))
+			continue
+		}
+		newStatus := fileInfo.Data.Status
+		if newStatus == "" {
+			continue
+		}
+		if newStatus != v.FsResourceStatus {
+			cfg.RewardBgVideos[i].FsResourceStatus = newStatus
+			updated = true
+			log.Info("checkin VideoCheckRS status changed",
+				log.Any("fsResourceId", v.FsResourceId),
+				log.Any("oldStatus", v.FsResourceStatus),
+				log.Any("newStatus", newStatus))
+		}
+	}
+
+	if updated {
+		err := checkinconfigmod.UpdateOne(
+			bson.M{"_id": cfg.ID},
+			bson.M{"$set": bson.M{"rewardBgVideos": cfg.RewardBgVideos}},
+		)
+		if err != nil {
+			log.Error("checkin VideoCheckRS update config fail", log.E(err))
+		}
+	}
+}

skd/job/dailyAdverCalc/job.go

@@ -0,0 +1,205 @@
+package dailyAdverCalc
+
+import (
+	"91porn-server/common"
+	"91porn-server/common/log"
+	"91porn-server/models/commod"
+	"91porn-server/models/s/dailyretentionmod"
+	"91porn-server/models/s/useradverstatmod"
+	"context"
+	"fmt"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+// retentionKey 用于内存中按 (RegDay, AdGroup) 分组
+type retentionKey struct {
+	RegDay  time.Time
+	AdGroup commod.AdGroup
+}
+
+// retentionAccum 内存中的累计数据
+type retentionAccum struct {
+	Retained   int64
+	AdClick    int64
+	TotalClick int64
+	PayCount   int64
+	PayTotal   int64
+}
+
+// CalcRetention 计算留存数据
+// targetDate: 活跃日期（要统计哪一天的活跃数据）
+// 例如：targetDate=15号，会计算所有在15号活跃的用户，按其注册日分组，计算各自的第N天留存
+func CalcRetention(ctx context.Context, targetDate time.Time) error {
+	targetDate = common.NormalizeDate(targetDate)
+	startDate := targetDate.AddDate(0, 0, -dailyretentionmod.MaxRetentionDays)
+	accumMap := make(map[retentionKey]*retentionAccum)
+
+	// 1. 在 Mongo 中按 (regDay, adGroup) 聚合，避免把当天明细全部拉回内存
+	stats, err := findUserAdStatSummaryByDate(ctx, targetDate, startDate)
+	if err != nil {
+		return fmt.Errorf("query UserAdStat failed: %w", err)
+	}
+	for _, stat := range stats {
+		key := retentionKey{
+			RegDay:  common.NormalizeDate(stat.ID.RegDay),
+			AdGroup: stat.ID.AdGroup,
+		}
+		if accumMap[key] == nil {
+			accumMap[key] = &retentionAccum{}
+		}
+		accumMap[key].Retained = stat.Retained
+		accumMap[key].AdClick = stat.AdClick
+		accumMap[key].TotalClick = stat.TotalClick
+		accumMap[key].PayCount = stat.PayCount
+		accumMap[key].PayTotal = stat.PayTotal
+	}
+	log.Info("calc retention", log.Any("stats.total", len(stats)), log.Any("stats", stats))
+	// 2. 批量写入 DailyRetention
+	now := time.Now()
+	writes := make([]mongo.WriteModel, 0, len(accumMap))
+	for key, acc := range accumMap {
+		// 计算是注册后第几天
+		dayN := int(targetDate.Sub(key.RegDay).Hours() / 24)
+		if dayN < 0 || dayN > dailyretentionmod.MaxRetentionDays {
+			log.Info("calc retention check dayN", log.Any("dayN", dayN))
+			continue // 超出范围，跳过
+		}
+
+		retained := acc.Retained
+
+		// 获取已有记录（用于计算留存率和累计值）
+		existingItem, _ := getDailyRetention(ctx, key.RegDay, key.AdGroup)
+		cohortSize := retained
+		if dayN > 0 && existingItem != nil {
+			cohortSize = existingItem.NewUsers
+		}
+		rate := calculateRetentionRate(retained, cohortSize)
+
+		// 计算累计值：前一天累计 + 当天值
+		prevAdClick, prevTotalClick, prevPayCount, prevPayTotal := getPrevAccum(existingItem, dayN)
+		adClickAcc := prevAdClick + acc.AdClick
+		totalClickAcc := prevTotalClick + acc.TotalClick
+		payCountAcc := prevPayCount + acc.PayCount
+		payTotalAcc := prevPayTotal + acc.PayTotal
+
+		writes = append(writes, buildDailyRetentionUpsertModel(
+			key.RegDay, key.AdGroup, dayN, retained, rate,
+			adClickAcc, totalClickAcc, payCountAcc, payTotalAcc, now,
+		))
+	}
+	log.Info("calc retention ", log.Any("writes", writes))
+	if err := dailyretentionmod.BulkWrite(writes, options.BulkWrite().SetOrdered(false)); err != nil {
+		return fmt.Errorf("bulk upsert daily retention failed: %w", err)
+	}
+
+	return nil
+}
+
+// getDailyRetention 获取某个 (RegDay, AdGroup) 的 DailyRetention 记录
+func getDailyRetention(ctx context.Context, regDay time.Time, adGroup commod.AdGroup) (*dailyretentionmod.DailyRetention, error) {
+	filter := bson.M{
+		"date":    regDay,
+		"adGroup": adGroup,
+	}
+	return dailyretentionmod.QueryOne(filter)
+}
+
+// buildDailyRetentionUpsertModel 构造留存数据 upsert 模型
+func buildDailyRetentionUpsertModel(regDay time.Time, adGroup commod.AdGroup, dayN int, retained int64, rate float64, adClick, totalClick, payCount, payTotal int64, now time.Time) mongo.WriteModel {
+	filter, update := buildDailyRetentionUpsertData(regDay, adGroup, dayN, retained, rate, adClick, totalClick, payCount, payTotal, now)
+	return mongo.NewUpdateOneModel().
+		SetFilter(filter).
+		SetUpdate(update).
+		SetUpsert(true)
+}
+
+func buildDailyRetentionUpsertData(regDay time.Time, adGroup commod.AdGroup, dayN int, retained int64, rate float64, adClick, totalClick, payCount, payTotal int64, now time.Time) (bson.M, bson.M) {
+	regDay = common.NormalizeDate(regDay)
+
+	filter := bson.M{
+		"date":    regDay,
+		"adGroup": adGroup,
+	}
+
+	// 使用点号表示法更新 map 中的特定 key
+	retainedKey := fmt.Sprintf("%s.%d", "userRetained", dayN)
+	ratesKey := fmt.Sprintf("%s.%d", "userRates", dayN)
+	adClickKey := fmt.Sprintf("%s.%d", "adClickAcc", dayN)
+	totalClickKey := fmt.Sprintf("%s.%d", "totalClickAcc", dayN)
+	payCountKey := fmt.Sprintf("%s.%d", "payCountAcc", dayN)
+	payTotalKey := fmt.Sprintf("%s.%d", "payTotalAcc", dayN)
+
+	update := bson.M{
+		"$setOnInsert": bson.M{
+			"date":      regDay,
+			"adGroup":   adGroup,
+			"createdAt": now,
+		},
+		"$set": bson.M{
+			"updatedAt":   now,
+			retainedKey:   retained,
+			ratesKey:      rate,
+			adClickKey:    adClick,
+			totalClickKey: totalClick,
+			payCountKey:   payCount,
+			payTotalKey:   payTotal,
+		},
+	}
+
+	// 如果是 day0，同时更新 NewUsers
+	if dayN == 0 {
+		update["$set"].(bson.M)["newUsers"] = retained
+	}
+	return filter, update
+}
+
+func calculateRetentionRate(retained, cohortSize int64) float64 {
+	if cohortSize <= 0 {
+		return 0
+	}
+	return float64(retained) / float64(cohortSize)
+}
+
+// getPrevAccum 获取前一天的累计值
+func getPrevAccum(item *dailyretentionmod.DailyRetention, dayN int) (adClick, totalClick, payCount, payTotal int64) {
+	if item == nil || dayN <= 0 {
+		return 0, 0, 0, 0
+	}
+	prevDay := dayN - 1
+	if item.AdClickAcc != nil {
+		adClick = item.AdClickAcc[prevDay]
+	}
+	if item.TotalClickAcc != nil {
+		totalClick = item.TotalClickAcc[prevDay]
+	}
+	if item.PayCountAcc != nil {
+		payCount = item.PayCountAcc[prevDay]
+	}
+	if item.PayTotalAcc != nil {
+		payTotal = item.PayTotalAcc[prevDay]
+	}
+	return
+}
+
+// findDailyRetentionByDateRange 查询指定日期范围内的 DailyRetention 记录
+func findDailyRetentionByDateRange(ctx context.Context, startDate, endDate time.Time) ([]*dailyretentionmod.DailyRetention, error) {
+	filter := bson.M{
+		"date": bson.M{
+			"$gte": startDate,
+			"$lte": endDate,
+		},
+	}
+	return dailyretentionmod.QueryAllList(filter)
+}
+
+// findUserAdStatSummaryByDate 查询指定日期在留存窗口内的聚合数据
+func findUserAdStatSummaryByDate(ctx context.Context, date, startDate time.Time) ([]*useradverstatmod.DailyRetentionSummary, error) {
+	local, _ := time.LoadLocation("Asia/Shanghai")
+	date = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, local)
+	startDate = common.NormalizeDate(startDate)
+	return useradverstatmod.AggregateDailyRetention(date, startDate, date)
+}

skd/job/dailyAdverCalc/job_test.go

@@ -0,0 +1,105 @@
+package dailyAdverCalc
+
+import (
+	"91porn-server/common"
+	"91porn-server/models/commod"
+	"91porn-server/models/s/dailyretentionmod"
+	"testing"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+func TestCalculateRetentionRate(t *testing.T) {
+	t.Run("zero cohort size returns zero", func(t *testing.T) {
+		if got := calculateRetentionRate(10, 0); got != 0 {
+			t.Fatalf("expected zero rate, got %v", got)
+		}
+	})
+
+	t.Run("uses retained divided by cohort size", func(t *testing.T) {
+		got := calculateRetentionRate(25, 100)
+		want := 0.25
+		if got != want {
+			t.Fatalf("expected %v, got %v", want, got)
+		}
+	})
+}
+
+func TestGetPrevAccum(t *testing.T) {
+	t.Run("nil item or day zero returns zeros", func(t *testing.T) {
+		adClick, totalClick, payCount, payTotal := getPrevAccum(nil, 0)
+		if adClick != 0 || totalClick != 0 || payCount != 0 || payTotal != 0 {
+			t.Fatalf("expected all zero values, got %d %d %d %d", adClick, totalClick, payCount, payTotal)
+		}
+	})
+
+	t.Run("reads previous day accumulators", func(t *testing.T) {
+		item := &dailyretentionmod.DailyRetention{
+			AdClickAcc:    map[int]int64{1: 3, 2: 8},
+			TotalClickAcc: map[int]int64{1: 5, 2: 13},
+			PayCountAcc:   map[int]int64{1: 1, 2: 2},
+			PayTotalAcc:   map[int]int64{1: 100, 2: 300},
+		}
+
+		adClick, totalClick, payCount, payTotal := getPrevAccum(item, 3)
+		if adClick != 8 || totalClick != 13 || payCount != 2 || payTotal != 300 {
+			t.Fatalf("unexpected previous accum values: %d %d %d %d", adClick, totalClick, payCount, payTotal)
+		}
+	})
+}
+
+func TestBuildDailyRetentionUpsertData(t *testing.T) {
+	loc, err := time.LoadLocation("Asia/Shanghai")
+	if err != nil {
+		t.Fatalf("load location failed: %v", err)
+	}
+	regDay := time.Date(2026, 3, 18, 14, 25, 0, 0, loc)
+	now := time.Date(2026, 3, 19, 9, 30, 0, 0, loc)
+
+	t.Run("day zero writes new users and normalizes date", func(t *testing.T) {
+		filter, update := buildDailyRetentionUpsertData(regDay, commod.AdGroupA, 0, 12, 1, 4, 8, 2, 600, now)
+
+		wantDate := common.NormalizeDate(regDay)
+		if got := filter["date"].(time.Time); !got.Equal(wantDate) {
+			t.Fatalf("expected normalized date %v, got %v", wantDate, got)
+		}
+		if got := filter["adGroup"]; got != commod.AdGroupA {
+			t.Fatalf("expected ad group %v, got %v", commod.AdGroupA, got)
+		}
+
+		setOnInsert := update["$setOnInsert"].(bson.M)
+		if got := setOnInsert["date"].(time.Time); !got.Equal(wantDate) {
+			t.Fatalf("expected setOnInsert date %v, got %v", wantDate, got)
+		}
+		if got := setOnInsert["createdAt"].(time.Time); !got.Equal(now) {
+			t.Fatalf("expected createdAt %v, got %v", now, got)
+		}
+
+		setFields := update["$set"].(bson.M)
+		if got := setFields["userRetained.0"]; got != int64(12) {
+			t.Fatalf("expected userRetained.0 to be 12, got %v", got)
+		}
+		if got := setFields["userRates.0"]; got != float64(1) {
+			t.Fatalf("expected userRates.0 to be 1, got %v", got)
+		}
+		if got := setFields["newUsers"]; got != int64(12) {
+			t.Fatalf("expected newUsers to be 12, got %v", got)
+		}
+	})
+
+	t.Run("non day zero does not overwrite new users", func(t *testing.T) {
+		_, update := buildDailyRetentionUpsertData(regDay, commod.AdGroupB, 2, 6, 0.3, 9, 11, 3, 800, now)
+
+		setFields := update["$set"].(bson.M)
+		if got := setFields["userRetained.2"]; got != int64(6) {
+			t.Fatalf("expected userRetained.2 to be 6, got %v", got)
+		}
+		if got := setFields["userRates.2"]; got != 0.3 {
+			t.Fatalf("expected userRates.2 to be 0.3, got %v", got)
+		}
+		if _, ok := setFields["newUsers"]; ok {
+			t.Fatalf("did not expect newUsers on non-day-zero update")
+		}
+	})
+}

skd/job/dataclean/dataclean.go

@@ -0,0 +1,29 @@
+package dataclean
+
+import (
+	"time"
+
+	"91porn-server/common/timeutil"
+)
+
+func RunSync() {
+	today := timeutil.BeginningOfDay(time.Now())
+
+	// log库清理
+	go clearVisitLog(today, 7)
+	go clearPlayLog(today)
+	go clearSearchLog(today)
+	go clearLoginTime(today)
+	go clearUserRegisterLog(today)
+
+	// stat库清理
+	go clearKeywordStat(today)
+	go clearNewsHot(today)
+	go clearKwRankStat(today)
+	go clearVideoStat(today)
+
+	// 主库清理
+	go clearNoticeFmt(today)
+	go clearUserAct(today)
+	go clearGuestUser(today)
+}

skd/job/dataclean/method.go

@@ -0,0 +1,94 @@
+package dataclean
+
+import (
+	"91porn-server/models/v/usermod"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/models/l/loginlgmod"
+	"91porn-server/models/l/playlgmod"
+	"91porn-server/models/l/registermod"
+	"91porn-server/models/l/searchlogmod"
+	"91porn-server/models/l/visitlogmod"
+	"91porn-server/models/s/hotstatmod"
+	"91porn-server/models/s/kwrankmod"
+	"91porn-server/models/s/kwstatmod"
+	"91porn-server/models/s/statvidmod"
+	"91porn-server/models/v/noticefmtmod"
+	"91porn-server/models/v/useractmod"
+)
+
+func clearVisitLog(today time.Time, day int) {
+	count, err := visitlogmod.DataClean(today.AddDate(0, 0, -day))
+	if err != nil {
+		log.Error("skd dataclean clearVisitLog error", log.E(err))
+		return
+	}
+	log.Info("skd dataclean clearVisitLog cleaned", log.Any("count", count))
+}
+
+func clearPlayLog(today time.Time) {
+	if err := playlgmod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -3)); err != nil {
+		log.Error("skd dataclean clearPlayLog error", log.E(err))
+	}
+}
+
+func clearSearchLog(today time.Time) {
+	if err := searchlogmod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -3)); err != nil {
+		log.Error("skd dataclean clearSearchLog error", log.E(err))
+	}
+}
+
+func clearKeywordStat(today time.Time) {
+	if err := kwstatmod.DeleteBeforeSumDate(nil, today.AddDate(0, 0, -30)); err != nil {
+		log.Error("skd dataclean clearKeywordStat error", log.E(err))
+	}
+}
+
+func clearNewsHot(today time.Time) {
+	if err := hotstatmod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -32)); err != nil {
+		log.Error("skd dataclean clearNewsHot error", log.E(err))
+	}
+}
+
+func clearLoginTime(today time.Time) {
+	if err := loginlgmod.DeleteBeforeLoginTime(nil, today.AddDate(0, 0, -5)); err != nil {
+		log.Error("skd dataclean clearLoginTime error", log.E(err))
+	}
+}
+
+func clearUserRegisterLog(today time.Time) {
+	if err := registermod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -7)); err != nil {
+		log.Error("skd dataclean clearUserRegisterLog error", log.E(err))
+	}
+}
+
+func clearKwRankStat(today time.Time) {
+	if err := kwrankmod.DeleteBeforeSumDate(nil, today.AddDate(0, 0, -30)); err != nil {
+		log.Error("skd dataclean clearKwRankStat error", log.E(err))
+	}
+}
+
+func clearVideoStat(today time.Time) {
+	if err := statvidmod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -7)); err != nil {
+		log.Error("skd dataclean clearVideoStat error", log.E(err))
+	}
+}
+
+func clearNoticeFmt(today time.Time) {
+	if err := noticefmtmod.DeleteBeforeCreatedAt(nil, today.AddDate(0, 0, -5)); err != nil {
+		log.Error("skd dataclean clearNoticeFmt error", log.E(err))
+	}
+}
+
+func clearUserAct(today time.Time) {
+	if err := useractmod.DeleteBeforeDailyDate(nil, today.AddDate(0, 0, -2)); err != nil {
+		log.Error("skd dataclean clearUserAct error", log.E(err))
+	}
+}
+
+func clearGuestUser(today time.Time) {
+	if err := usermod.DeleteGuestBeforeDate(nil, today.AddDate(0, 0, -7)); err != nil {
+		log.Error("skd dataclean clearGuestUser error", log.E(err))
+	}
+}

skd/job/elastic/esaudiobook/method.go

@@ -0,0 +1,27 @@
+package esaudiobook
+
+import (
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/models"
+	"91porn-server/models/v/audiobookmod"
+)
+
+// 同步用户数据
+func SyncDataToES(data []audiobookmod.AudioBookBase) error {
+	if len(data) == 0 {
+		return nil
+	}
+	var source = elastic.M{}
+	for k := range data {
+		var tmp = data[k]
+		source[data[k].ID.Hex()] = tmp
+	}
+	log.Info("Elastic sync audiobook data committing...")
+	if err := es.Bulk(models.ESInfoAudioBookTable, source); err != nil {
+		log.Info("Elastic sync audiobook data commit failed.")
+		return err
+	}
+	log.Info("Elastic sync audiobook data committed.")
+	return nil
+}

skd/job/elastic/esaudiobook/sync.go

@@ -0,0 +1,58 @@
+package esaudiobook
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/v/audiobookmod"
+	"91porn-server/skd/skdg"
+)
+
+var es *elastic.Client
+
+func RunSync(defaultDate time.Time) {
+	es = skdg.VideoES
+	log.Info("Elastic Search audiobook sync start...")
+	defer log.Info("Elastic Search audiobook sync end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncAudioBookJob, defaultDate)
+	if err != nil {
+		log.Error("Elastic sync audiobook data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now().Add(-time.Minute * 20)
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		data, err := audiobookmod.GetListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail)
+		if err != nil {
+			return
+		}
+		if err = commit(data, subTimeRange.Tail); err != nil {
+			log.Error("Elastic sync audiobook data RunSync commit err! ", log.E(err))
+			return
+		}
+		lastSyncAt = end
+	}
+}
+
+func commit(data []audiobookmod.AudioBookBase, syncTime time.Time) error {
+	if err := SyncDataToES(data); err != nil {
+		return fmt.Errorf("Elastic sync commit audiobook data SyncDataToES failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncAudioBookJob, syncTime); err != nil {
+		return fmt.Errorf("Elastic sync commit audiobook data UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}

skd/job/elastic/eskeyword/method.go

@@ -0,0 +1,26 @@
+package eskeyword
+
+import (
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/models"
+	"91porn-server/models/s/kwstatmod"
+)
+
+// 同步数据到ES
+func SyncDataToES(data []kwstatmod.Keyword) error {
+	if len(data) == 0 {
+		return nil
+	}
+	var source = elastic.M{}
+	for _, v := range data {
+		source[v.ID.Hex()] = kwstatmod.ESKeywordSrc(v)
+	}
+	log.Info("Elastic sync keyword committing...")
+	if err := es.Bulk(models.ESStatKeywordTable, source); err != nil {
+		log.Info("Elastic sync keyword commit failed.")
+		return err
+	}
+	log.Info("Elastic sync keyword committed.")
+	return nil
+}

skd/job/elastic/eskeyword/sync.go

@@ -0,0 +1,58 @@
+package eskeyword
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/kwstatmod"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/skd/skdg"
+)
+
+var es *elastic.Client
+
+func RunSync(defaultDate time.Time) {
+	es = skdg.VideoES
+	log.Info("Elastic Search sync keyword start...")
+	defer log.Info("Elastic Search tag sync keyword end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncKeywordJob, defaultDate)
+	if err != nil {
+		log.Error("Elastic sync keyword data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now()
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		data, err := kwstatmod.GetKeywordsByTimeRange(subTimeRange.Head, subTimeRange.Tail)
+		if err != nil {
+			return
+		}
+		if err = commit(data, subTimeRange.Tail); err != nil {
+			log.Error("Elastic sync keyword data RunSync commit err! ", log.E(err))
+			return
+		}
+		lastSyncAt = end
+	}
+}
+
+func commit(data []kwstatmod.Keyword, syncTime time.Time) error {
+	if err := SyncDataToES(data); err != nil {
+		return fmt.Errorf("Elastic sync commit keyword data SyncDataToES failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncKeywordJob, syncTime); err != nil {
+		return fmt.Errorf("Elastic sync commit keyword data UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}

skd/job/elastic/esmedia/method.go

@@ -0,0 +1,65 @@
+package esmedia
+
+import (
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/models"
+	"91porn-server/models/v/mediamod"
+)
+
+// SyncDataToES 同步数据到ES
+func SyncDataToES(data []*mediamod.Media) error {
+	if len(data) == 0 {
+		return nil
+	}
+
+	source := buildSource(data)
+	log.Info("Elastic sync ACG media data committing...")
+	if err := es.BulkChecked(models.ESMediaTable, source); err != nil {
+		log.Info("Elastic sync ACG media data commit failed.")
+		return err
+	}
+	log.Info("Elastic sync ACG media data committed.")
+	return nil
+}
+
+func buildSource(data []*mediamod.Media) elastic.M {
+	var source = elastic.M{}
+	for _, v := range data {
+		source[v.ID.Hex()] = mediamod.ESMedia{
+			ID:                 v.ID,
+			Title:              v.Title,
+			HorizontalCover:    v.HorizontalCover,
+			VerticalCover:      v.VerticalCover,
+			Tags:               v.Tags,
+			Summary:            v.Summary,
+			TotalEpisode:       v.TotalEpisode,
+			UpdateStatus:       v.UpdateStatus,
+			MediaType:          v.MediaType,
+			Kind:               v.Kind,
+			PermissionIconHide: v.PermissionIconHide,
+			Permission:         v.Permission,
+			Price:              v.Price,
+			Direction:          v.Direction,
+			FreeTime:           v.FreeTime,
+			CountLike:          v.CountLike,
+			CountBrowse:        v.CountBrowse,
+			CountCollect:       v.CountCollect,
+			MID:                v.MID,
+			ModuleName:         v.ModuleName,
+			SectionName:        v.SectionName,
+			SID:                v.SID,
+			SectionSort:        v.SectionSort,
+			Choice:             v.Choice,
+			ChoiceSort:         v.ChoiceSort,
+			Number:             v.Number,
+			SortCode:           v.SortCode,
+			Status:             v.Status,
+			IsDelete:           v.IsDelete,
+			UpdatedAct:         v.UpdatedAct,
+			CreatedAt:          v.CreatedAt,
+			UpdateTime:         v.UpdateTime,
+		}
+	}
+	return source
+}

skd/job/elastic/esmedia/method_test.go

@@ -0,0 +1,25 @@
+package esmedia
+
+import (
+	"testing"
+
+	"91porn-server/models/v/mediamod"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func TestBuildSourceIncludesCountCollect(t *testing.T) {
+	id := primitive.NewObjectID()
+	source := buildSource([]*mediamod.Media{{
+		ID:           id,
+		CountCollect: 12,
+	}})
+
+	media, ok := source[id.Hex()].(mediamod.ESMedia)
+	if !ok {
+		t.Fatalf("source type = %T, want mediamod.ESMedia", source[id.Hex()])
+	}
+	if media.CountCollect != 12 {
+		t.Fatalf("countCollect = %d, want 12", media.CountCollect)
+	}
+}

skd/job/elastic/esmedia/sync.go

@@ -0,0 +1,80 @@
+package esmedia
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/v/mediamod"
+	"91porn-server/skd/skdg"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+var es *elastic.Client
+
+func RunSync(defaultDate time.Time) {
+	es = skdg.VideoES
+	log.Info("Elastic Search ACG media sync start...")
+	defer log.Info("Elastic Search ACG media sync end.")
+	// MongoDB 日期精度为毫秒；本轮开始后的修改必须留给下一轮再次同步。
+	startedAt := time.Now().Truncate(time.Millisecond)
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncMediaJob, defaultDate)
+	if err != nil {
+		log.Error("Elastic sync ACG media get last record time failed", log.E(err))
+		return
+	}
+	if !startedAt.After(lastSyncAt) {
+		return
+	}
+	// 固定 ID 上界，避免持续新增数据使本轮无法结束。
+	maxID, err := mediamod.GetLatestIDForESSync()
+	if err != nil {
+		log.Error("Elastic sync ACG media get latest ID failed", log.E(err))
+		return
+	}
+	sync := mediaSync{
+		read:  mediamod.GetListForESSync,
+		write: SyncDataToES,
+		checkpoint: func(at time.Time) error {
+			return statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncMediaJob, at)
+		},
+	}
+	if err = sync.run(lastSyncAt, startedAt, maxID); err != nil {
+		log.Error("Elastic sync ACG media failed; checkpoint not confirmed", log.E(err))
+	}
+}
+
+type mediaSync struct {
+	read       func(time.Time, primitive.ObjectID, primitive.ObjectID, int) ([]*mediamod.Media, error)
+	write      func([]*mediamod.Media) error
+	checkpoint func(time.Time) error
+}
+
+func (s mediaSync) run(since, startedAt time.Time, maxID primitive.ObjectID) error {
+	const size = 1000
+	var after primitive.ObjectID
+	for !maxID.IsZero() {
+		data, err := s.read(since, after, maxID, size)
+		if err != nil {
+			return fmt.Errorf("read media batch: %w", err)
+		}
+		if len(data) == 0 {
+			break
+		}
+		if err = s.write(data); err != nil {
+			return fmt.Errorf("write media batch: %w", err)
+		}
+		after = data[len(data)-1].ID
+		if len(data) < size || after == maxID {
+			break
+		}
+	}
+	// 所有批次及每条 ES 写入均成功后才推进；失败时下一轮从原进度幂等重试。
+	if err := s.checkpoint(startedAt); err != nil {
+		return fmt.Errorf("save media checkpoint: %w", err)
+	}
+	return nil
+}

skd/job/elastic/essearchlog/method.go

@@ -0,0 +1,26 @@
+package essearchlog
+
+import (
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/models"
+	"91porn-server/models/l/searchlogmod"
+)
+
+// 同步数据到ES
+func SyncDataToES(data []searchlogmod.SearchLog) error {
+	if len(data) == 0 {
+		return nil
+	}
+	var source = elastic.M{}
+	for _, v := range data {
+		source[v.ID.Hex()] = searchlogmod.ESSearchLogSource(v)
+	}
+	log.Info("Elastic sync search log data committing...")
+	if err := es.Bulk(models.ESLogSearchLogTable, source); err != nil {
+		log.Info("Elastic sync search log data commit failed.")
+		return err
+	}
+	log.Info("Elastic sync search log data committed.")
+	return nil
+}

skd/job/elastic/essearchlog/sync.go

@@ -0,0 +1,58 @@
+package essearchlog
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/l/searchlogmod"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/skd/skdg"
+)
+
+var es *elastic.Client
+
+func RunSync(defaultDate time.Time) {
+	es = skdg.VideoES
+	log.Info("Elastic Search sync search log start...")
+	defer log.Info("Elastic Search tag sync search log end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncSearchLogJob, defaultDate)
+	if err != nil {
+		log.Error("Elastic sync search log data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now()
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		data, err := searchlogmod.GetSearchLogByTimeRange(subTimeRange.Head, subTimeRange.Tail)
+		if err != nil {
+			return
+		}
+		if err = commit(data, subTimeRange.Tail); err != nil {
+			log.Error("Elastic sync search log data RunSync commit err! ", log.E(err))
+			return
+		}
+		lastSyncAt = end
+	}
+}
+
+func commit(data []searchlogmod.SearchLog, syncTime time.Time) error {
+	if err := SyncDataToES(data); err != nil {
+		return fmt.Errorf("Elastic sync commit search log data SyncDataToES failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncSearchLogJob, syncTime); err != nil {
+		return fmt.Errorf("Elastic sync commit search log data UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}

skd/job/elastic/estag/method.go

@@ -0,0 +1,44 @@
+package estag
+
+import (
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/models"
+	"91porn-server/models/v/tagmod"
+)
+
+// 同步数据到ES
+func SyncDataToES(data []tagmod.Tag) error {
+	if len(data) == 0 {
+		return nil
+	}
+	var source = elastic.M{}
+	for _, v := range data {
+		var tmp = tagmod.ESTag{
+			ID:             v.ID,
+			TagName:        v.TagName,
+			SortCode:       v.SortCode,
+			VPlayCount:     v.VPlayCount,
+			TPlayCount:     v.TPlayCount,
+			FakePlayCount:  v.FakePlayCount,
+			PlayRating:     v.PlayRating,
+			CollCount:      v.CollCount,
+			LikeCount:      v.LikeCount,
+			FakeCollCount:  v.FakeCollCount,
+			VideoCount:     v.VideoCount,
+			IsActive:       v.IsActive,
+			IsWonder:       v.IsWonder,
+			WonderSortCode: v.WonderSortCode,
+			CreatedAt:      v.CreatedAt,
+			UpdatedAt:      v.UpdatedAt,
+		}
+		source[v.ID.Hex()] = tmp
+	}
+	log.Info("Elastic sync tag data committing...")
+	if err := es.Bulk(models.ESInfoTagTable, source); err != nil {
+		log.Info("Elastic sync tag data commit failed.")
+		return err
+	}
+	log.Info("Elastic sync tag data committed.")
+	return nil
+}

skd/job/elastic/estag/sync.go

@@ -0,0 +1,61 @@
+package estag
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/v/tagmod"
+	"91porn-server/skd/skdg"
+)
+
+var es *elastic.Client
+
+func RunSync(defaultDate time.Time) {
+	es = skdg.VideoES
+	log.Info("Elastic Search tag sync start...")
+	defer log.Info("Elastic Search tag sync end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncTagJob, defaultDate)
+	if err != nil {
+		log.Error("Elastic sync tag data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now()
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		data, err := tagmod.GetTagListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail)
+		if err != nil {
+			return
+		}
+		if err = commit(data, subTimeRange.Tail); err != nil {
+			log.Error("Elastic sync tag data RunSync commit err! ", log.E(err))
+			return
+		}
+		lastSyncAt = end
+	}
+}
+
+func commit(data []tagmod.Tag, syncTime time.Time) error {
+	if len(data) <= 0 {
+		return nil
+	}
+	if err := SyncDataToES(data); err != nil {
+		return fmt.Errorf("Elastic sync commit tag data SyncDataToES failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncTagJob, syncTime); err != nil {
+		return fmt.Errorf("Elastic sync commit tag data UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}

skd/job/elastic/esuser/method.go

@@ -0,0 +1,38 @@
+package esuser
+
+import (
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/models"
+	"91porn-server/models/v/usermod"
+)
+
+// 同步用户数据
+func SyncDataToES(data []usermod.User) error {
+	if len(data) == 0 {
+		return nil
+	}
+	var source = elastic.M{}
+	for _, v := range data {
+		var tmp = usermod.ESUser{
+			ID:        v.ID,
+			UID:       v.UID,
+			Gender:    v.Gender,
+			Name:      v.Name,
+			Summary:   v.Summary,
+			Region:    v.Region,
+			CreatedAt: v.CreatedAt,
+			UpdatedAt: v.UpdatedAt,
+			HasLocked: v.HasLocked,
+			HasBanned: v.HasBanned,
+		}
+		source[v.ID.Hex()] = tmp
+	}
+	log.Info("Elastic sync user data committing...")
+	if err := es.Bulk(models.ESInfoUserTable, source); err != nil {
+		log.Info("Elastic sync user data commit failed.")
+		return err
+	}
+	log.Info("Elastic sync user data committed.")
+	return nil
+}

skd/job/elastic/esuser/sync.go

@@ -0,0 +1,72 @@
+package esuser
+
+import (
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/v/usermod"
+	"91porn-server/skd/skdg"
+	"fmt"
+	"time"
+)
+
+var es *elastic.Client
+
+func RunSync(defaultDate time.Time) {
+	es = skdg.VideoES
+	log.Info("Elastic Search user sync start...")
+	defer log.Info("Elastic Search user sync end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncUserJob, defaultDate)
+	if err != nil {
+		log.Error("Elastic sync user data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now()
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		var page, size = 1, 1000
+		hasNext := true
+		var data []usermod.User
+		for hasNext {
+			data, hasNext, err = usermod.GetUserListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
+			if err != nil {
+				log.Error("Elastic sync user data GetUserListByUpdateTimeRange err! ", log.E(err))
+				return
+			}
+
+			if hasNext && len(data) > 0 {
+				_ = SyncDataToES(data)
+				page++
+			} else {
+				if err = commit(data, subTimeRange.Tail); err != nil {
+					log.Error("Elastic sync user data RunSync commit err! ", log.E(err))
+					return
+				}
+			}
+		}
+
+		lastSyncAt = end
+	}
+}
+
+func commit(data []usermod.User, syncTime time.Time) error {
+	err := SyncDataToES(data)
+	if err != nil {
+		return fmt.Errorf("Elastic sync commit user data SyncDataToES failed!, err:%+v\n", err)
+	}
+	err = statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncUserJob, syncTime)
+	if err != nil {
+		return fmt.Errorf("Elastic sync commit user data UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return err
+}

skd/job/elastic/esvideo/method.go

@@ -0,0 +1,108 @@
+package esvideo
+
+import (
+	"math"
+	"time"
+
+	"91porn-server/common/elastic"
+	"91porn-server/common/localcache"
+	"91porn-server/common/log"
+	"91porn-server/models"
+	"91porn-server/models/v/tagmod"
+	"91porn-server/models/v/vidmod"
+	"91porn-server/models/v/vidpopmod"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+const vidPopCfgKey = "vidPopCfgKey"
+
+// SyncDataToES 同步数据到ES
+func SyncDataToES(data []*vidmod.VideoModel) error {
+	if len(data) == 0 {
+		return nil
+	}
+	// 获取标签名字
+	var tagsID []primitive.ObjectID
+	for _, v := range data {
+		tagsID = append(tagsID, v.Tags...)
+	}
+	tags, err := tagmod.FindTagsByIDS(tagsID)
+	if err != nil {
+		return err
+	}
+	var tagsNameMap = make(map[primitive.ObjectID]string)
+	for _, v := range tags {
+		tagsNameMap[v.ID] = v.TagName
+	}
+	var source = elastic.M{}
+	for _, v := range data {
+		var tagsName []string
+		for _, v1 := range v.Tags {
+			tagsName = append(tagsName, tagsNameMap[v1])
+		}
+		source[v.ID.Hex()] = vidmod.ESVideo{
+			ID:                 v.ID,
+			PublisherID:        v.PublisherID,
+			Title:              v.Title,
+			NewsType:           v.NewsType,
+			Tags:               v.Tags,
+			TagsName:           tagsName,
+			Filename:           v.Filename,
+			PlayCount:          v.PlayCount,
+			EffectivePlayCount: v.EffectivePlayCount,
+			PurchaseCount:      v.PurchaseCount,
+			LikeCount:          v.LikeCount,
+			CommentCount:       v.CommentCount,
+			CollectCount:       v.CollectCount,
+			ShareCount:         v.ShareCount,
+			FakeLikeCount:      v.FakeLikeCount,
+			FakeShareCount:     v.FakeShareCount,
+			FakePlayCount:      v.FakePlayCount,
+			Status:             v.Status,
+			Location:           v.Location,
+			CreatedAt:          v.CreatedAt,
+			UpdatedAt:          v.UpdatedAt,
+			ReviewedAt:         v.ReviewAt,
+			Hot:                calcHot(v),
+			PlayTime:           v.PlayTime,
+			Coins:              v.Coins,
+		}
+	}
+	log.Info("Elastic sync video data committing...")
+	if err = es.Bulk(models.ESInfoVideoTable, source); err != nil {
+		log.Info("Elastic sync video data commit failed.")
+		return err
+	}
+	log.Info("Elastic sync video data committed.")
+	return nil
+}
+
+func calcHot(video *vidmod.VideoModel) float64 {
+	if video.Status == vidmod.CheckPass || video.Status == vidmod.Free {
+		cfg, err := getVideoPopularityConfig()
+		if err != nil {
+			log.Error("Elastic sync video getVideoPopularityConfig", log.E(err))
+			return 0
+		}
+		qualityScore := float64(video.PlayCount + cfg.EffectivePlayCountMultiplier*video.EffectivePlayCount +
+			cfg.LikeCountMultiplier*video.LikeCount)
+		hot := (qualityScore + float64(cfg.InitialPopularity)) / math.Pow(1+time.Since(video.ReviewAt).Hours(),
+			cfg.ReviewTimePower)
+		return hot
+	}
+	return 0
+}
+
+func getVideoPopularityConfig() (*vidpopmod.VideoPopularityConfig, error) {
+	cfg, ok := localcache.C.Get(vidPopCfgKey)
+	if ok {
+		return cfg.(*vidpopmod.VideoPopularityConfig), nil
+	}
+	res, err := vidpopmod.FindOne()
+	if err != nil {
+		return nil, err
+	}
+	localcache.C.Set(vidPopCfgKey, &res, 5*time.Minute)
+	return &res, nil
+}

skd/job/elastic/esvideo/sync.go

@@ -0,0 +1,74 @@
+package esvideo
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/elastic"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/v/vidmod"
+	"91porn-server/skd/skdg"
+)
+
+var es *elastic.Client
+
+func RunSync(defaultDate time.Time) {
+	es = skdg.VideoES
+	log.Info("Elastic Search video sync start...")
+	defer log.Info("Elastic Search video sync end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.ElasticSyncVideoJob, defaultDate)
+	if err != nil {
+		log.Error("Elastic sync video data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now()
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		var page, size = 1, 1000
+		hasNext := true
+		var data []*vidmod.VideoModel
+		for hasNext {
+			data, hasNext, err = vidmod.GetVideoListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
+			if err != nil {
+				log.Error("Elastic sync video data GetVideoListByUpdateTimeRange err! ", log.E(err))
+				return
+			}
+			if hasNext && len(data) > 0 {
+				_ = SyncDataToES(data)
+				page++
+			} else {
+				if err = commit(data, subTimeRange.Tail); err != nil {
+					log.Error("Elastic sync video data RunSync commit err! ", log.E(err))
+					return
+				}
+			}
+		}
+
+		lastSyncAt = end
+	}
+}
+
+func commit(data []*vidmod.VideoModel, syncTime time.Time) error {
+	if len(data) <= 0 {
+		_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncVideoJob, syncTime)
+		return nil
+	}
+	if err := SyncDataToES(data); err != nil {
+		return fmt.Errorf("Elastic sync commit video data SyncDataToES failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.ElasticSyncVideoJob, syncTime); err != nil {
+		return fmt.Errorf("Elastic sync commit video data UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}

skd/job/elastic/rdvideo/method.go

@@ -0,0 +1,27 @@
+package rdvideo
+
+import (
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/log"
+	"91porn-server/common/redis"
+	"91porn-server/models/v/vidmod"
+)
+
+// SyncDataToRedis 同步数据到redis
+func SyncDataToRedis(data []*vidmod.VideoModel) error {
+	if len(data) == 0 {
+		return nil
+	}
+
+	for _, v := range data {
+		if v.NewsType != vidmod.SHORT || (v.Status != vidmod.CheckPass && v.Status != vidmod.Free) {
+			// 去除
+			_, _ = redis.Handler.SRem(redisconst.ShortVideosKey, v.ID.Hex())
+		} else {
+			_, _ = redis.Handler.SAdd(redisconst.ShortVideosKey, v.ID.Hex())
+		}
+	}
+
+	log.Info("Redis sync video data completed.")
+	return nil
+}

skd/job/elastic/rdvideo/sync.go

@@ -0,0 +1,77 @@
+package rdvideo
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/log"
+	"91porn-server/common/redis"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/v/vidmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("Redis short video sync start...")
+	defer log.Info("Redis short video sync end.")
+	// 如果短视频缓存存在的数据量过低，表示缓存被清理过，需要重新同步
+	c, err := redis.Handler.SCard(redisconst.ShortVideosKey)
+	if err == nil && c < 1000 {
+		_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, defaultDate)
+	}
+
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.RedisSyncShortVideoJob, defaultDate)
+	if err != nil {
+		log.Error("Redis sync short video data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now()
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.OneDayScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.OneDayScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		var page, size = 1, 1000
+		hasNext := true
+		var data []*vidmod.VideoModel
+		for hasNext {
+			data, hasNext, err = vidmod.GetShortVideoListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
+			if err != nil {
+				log.Error("Redis sync short video data GetVideoListByUpdateTimeRange err! ", log.E(err))
+				return
+			}
+			if hasNext && len(data) > 0 {
+				_ = SyncDataToRedis(data)
+				page++
+			} else {
+				if err = commit(data, subTimeRange.Tail); err != nil {
+					log.Error("Redis sync short video data RunSync commit err! ", log.E(err))
+					return
+				}
+			}
+		}
+
+		lastSyncAt = end
+	}
+}
+
+func commit(data []*vidmod.VideoModel, syncTime time.Time) error {
+	if len(data) <= 0 {
+		_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime)
+		return nil
+	}
+	if err := SyncDataToRedis(data); err != nil {
+		return fmt.Errorf("Redis sync commit short video data SyncDataToES failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime); err != nil {
+		return fmt.Errorf("Redis sync commit short video data UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}

skd/job/hevc_transcode.go

@@ -0,0 +1,555 @@
+package job
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"91porn-server/common/constant"
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/hevcpull"
+	"91porn-server/common/laosiji"
+	"91porn-server/common/log"
+	"91porn-server/models/v/hevctaskmod"
+	"91porn-server/models/v/moduleconfmod"
+	"91porn-server/models/v/modulesectionmod"
+	"91porn-server/models/v/modulevidmod"
+	"91porn-server/models/v/sysconfmod"
+	"91porn-server/models/v/vidmod"
+	"91porn-server/skd/skdg"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+const (
+	h265DefaultMaxRunning = 400 // 配置缺失或无效时，最多允许 400 个视频处于云端转码中
+	h265CheckBatchSize    = 100 // 每轮最多轮询 100 个已提交的 H265 任务
+	h265HomeCandidateSize = 400 // 首页每个模块、每种排序最多保障 400 个视频
+	h265PendingTimeout    = 240 * time.Hour
+	h265PullURLTTL        = 480 * time.Hour
+)
+
+var (
+	h265SubmitRunning int32
+	h265CheckRunning  int32
+	h265ScanRunning   int32
+
+	h265MissingCredentialsLogOnce sync.Once
+	h265MissingAppURLLogOnce      sync.Once
+	h265InvalidPullSecretLogOnce  sync.Once
+)
+
+// h265TranscodingJobEnabled 默认只允许生产环境执行真实云转码。
+// 测试环境必须通过 hevc.enableTranscode 显式开启，避免误提交云端任务。
+func h265TranscodingJobEnabled() bool {
+	if skdg.Conf == nil {
+		return false
+	}
+	log.Info("h265TranscodingJobEnabled ", log.Any("skdg.Conf.Base.Env", skdg.Conf.Base.Env))
+	log.Info("h265TranscodingJobEnabled ", log.Any("skdg.Conf.Hevc.EnableTranscode", skdg.Conf.Hevc.EnableTranscode))
+	if skdg.Conf.Hevc.EnableTranscode != nil && !*skdg.Conf.Hevc.EnableTranscode {
+		return false
+	}
+	if skdg.Conf.Base.Env != constant.ProdEnv &&
+		(skdg.Conf.Hevc.EnableTranscode == nil || !*skdg.Conf.Hevc.EnableTranscode) {
+		return false
+	}
+	if !laosiji.Configured() {
+		h265MissingCredentialsLogOnce.Do(func() {
+			log.Warn("H265 transcoding jobs disabled: laosiji credentials are incomplete")
+		})
+		return false
+	}
+	if _, err := hevcpull.SignURL(
+		"https://configuration-check.invalid/source.m3u8",
+		skdg.Conf.Hevc.PullSecret,
+		time.Now().Add(h265PullURLTTL),
+	); err != nil {
+		h265InvalidPullSecretLogOnce.Do(func() {
+			log.Warn("H265 transcoding jobs disabled: hevc.pullSecret is missing or invalid", log.E(err))
+		})
+		return false
+	}
+	if buildH265AppTranscodeM3u8URL(
+		skdg.Conf.Url.AppApiUrl,
+		"health-check.m3u8",
+		skdg.Conf.Hevc.PullSecret,
+		time.Now().Add(h265PullURLTTL),
+	) == "" {
+		h265MissingAppURLLogOnce.Do(func() {
+			log.Warn("H265 transcoding jobs disabled: url.appApiUrl is missing or invalid", log.Any("skdg.Conf.Url.AppApiUrl", skdg.Conf.Url.AppApiUrl))
+		})
+		return false
+	}
+	return true
+}
+
+// SubmitH265TranscodingQueue 把本地等待队列中的任务异步提交到云转码。
+func SubmitH265TranscodingQueue() {
+	if !h265TranscodingJobEnabled() {
+		return
+	}
+	if !atomic.CompareAndSwapInt32(&h265SubmitRunning, 0, 1) {
+		log.Info("SubmitH265TranscodingQueue skip: previous run still running")
+		return
+	}
+	defer atomic.StoreInt32(&h265SubmitRunning, 0)
+
+	ctx := context.Background()
+	maxRunning := h265MaxRunning()
+	pendingCount, err := vidmod.CountH265Pending()
+	if err != nil {
+		return
+	}
+	available := int64(maxRunning) - pendingCount
+	if available <= 0 {
+		return
+	}
+
+	domains, err := laosiji.SystemDomains(ctx)
+	if err != nil {
+		log.Warn("SubmitH265TranscodingQueue skip: laosiji domains failed", log.E(err))
+		return
+	}
+	videos, err := vidmod.GetQueuedH265Videos(available)
+	if err != nil {
+		return
+	}
+	for _, video := range videos {
+		if err = submitH265Video(ctx, domains, video, maxRunning); err != nil {
+			log.Warn("SubmitH265TranscodingQueue submit failed", log.Any("videoId", video.ID), log.E(err))
+			if errors.Is(err, laosiji.ErrTranscodeQueueFull) {
+				return
+			}
+		}
+	}
+}
+
+// CheckH265TranscodingResults 轮询云端任务并回写 h265Url。
+func CheckH265TranscodingResults() {
+	if !h265TranscodingJobEnabled() {
+		return
+	}
+	if !atomic.CompareAndSwapInt32(&h265CheckRunning, 0, 1) {
+		log.Info("CheckH265TranscodingResults skip: previous run still running")
+		return
+	}
+	defer atomic.StoreInt32(&h265CheckRunning, 0)
+
+	ctx := context.Background()
+	domains, err := laosiji.SystemDomains(ctx)
+	if err != nil {
+		log.Warn("CheckH265TranscodingResults skip: laosiji domains failed", log.E(err))
+		return
+	}
+	videos, err := vidmod.GetPendingH265Videos(h265CheckBatchSize)
+	if err != nil || len(videos) == 0 {
+		return
+	}
+	now := time.Now()
+	for _, video := range videos {
+		if video == nil {
+			continue
+		}
+		if h265PendingTimedOut(video, now) {
+			timeoutMsg := fmt.Sprintf("H265 transcode pending timeout after %.0fh", h265PendingTimeout.Hours())
+			marked, markErr := vidmod.MarkH265Failed(video.ID, video.SourceURL, video.H265PendingAt)
+			if markErr != nil {
+				log.Warn("CheckH265TranscodingResults mark timeout failed", log.Any("videoId", video.ID), log.E(markErr))
+			} else if marked {
+				upsertH265TaskWithError(video, domains, "", hevctaskmod.H265TaskStatusFailed, timeoutMsg)
+			} else {
+				log.Info("CheckH265TranscodingResults ignored stale timeout", log.Any("videoId", video.ID))
+			}
+			continue
+		}
+		result, found, queryErr := queryH265Transcode(ctx, domains, video)
+		if queryErr != nil {
+			log.Warn("CheckH265TranscodingResults query failed", log.Any("videoId", video.ID), log.E(queryErr))
+			// 保留原 processing 审计状态。查询返回和日志已经足够诊断，
+			// 此处不落库可避免并发成功后被旧查询错误覆盖。
+			continue
+		}
+		if !found {
+			const notFoundMsg = "laosiji transcode task not found"
+			log.Warn("CheckH265TranscodingResults task not found, retry as failed", log.Any("videoId", video.ID))
+			marked, markErr := vidmod.MarkH265Failed(video.ID, video.SourceURL, video.H265PendingAt)
+			if markErr != nil {
+				log.Warn("CheckH265TranscodingResults mark not-found failed", log.Any("videoId", video.ID), log.E(markErr))
+			} else if marked {
+				// Failed 状态仍会进入等待队列，但会累计失败次数，避免云端永久查无任务时无限重提。
+				upsertH265TaskWithError(video, domains, "", hevctaskmod.H265TaskStatusFailed, notFoundMsg)
+			} else {
+				log.Info("CheckH265TranscodingResults ignored stale not-found result", log.Any("videoId", video.ID))
+			}
+			continue
+		}
+		applyH265Result(video, result)
+	}
+}
+
+func h265PendingTimedOut(video *vidmod.VideoModel, now time.Time) bool {
+	if video == nil ||
+		video.H265Status != vidmod.H265StatusPending ||
+		video.H265PendingAt.IsZero() ||
+		video.H265PendingAt.After(now) {
+		return false
+	}
+	return now.Sub(video.H265PendingAt) >= h265PendingTimeout
+}
+
+// ScanHomeH265TranscodingQueue 将首页各启用视频模块的重点 SP 视频加入等待队列。
+func ScanHomeH265TranscodingQueue() {
+	if !h265TranscodingJobEnabled() {
+		return
+	}
+	if !atomic.CompareAndSwapInt32(&h265ScanRunning, 0, 1) {
+		log.Info("ScanHomeH265TranscodingQueue skip: previous run still running")
+		return
+	}
+	defer atomic.StoreInt32(&h265ScanRunning, 0)
+
+	moduleIDs, moduleIDHexes := h265HomeModuleIDs()
+	if len(moduleIDs) == 0 {
+		return
+	}
+
+	videoIDs := make([]primitive.ObjectID, 0)
+	seen := make(map[primitive.ObjectID]struct{})
+	for _, moduleIDHex := range moduleIDHexes {
+		for _, sortType := range []int{1, 2, 3, 7, 9} {
+			videos, err := vidmod.GetVideoListByCond(homeH265Filter(moduleIDHex, sortType), homeH265Options(sortType))
+			if err != nil {
+				log.Warn("ScanHomeH265TranscodingQueue video query failed",
+					log.Any("moduleId", moduleIDHex), log.Any("sortType", sortType), log.E(err))
+				continue
+			}
+			appendUniqueH265VideoIDs(&videoIDs, seen, videos)
+		}
+	}
+
+	for _, moduleID := range moduleIDs {
+		sections, err := modulesectionmod.GetAllBySubModuleID(moduleID)
+		if err != nil {
+			log.Warn("ScanHomeH265TranscodingQueue section query failed", log.Any("moduleId", moduleID), log.E(err))
+			continue
+		}
+		for _, section := range sections {
+			sectionVideos, err := modulevidmod.SectionVideosBySectionID(section.ID, options.Find().
+				SetLimit(h265HomeCandidateSize).
+				SetSort(bson.D{{Key: "sortCode", Value: -1}, {Key: "videoReviewedAt", Value: -1}}))
+			if err != nil {
+				log.Warn("ScanHomeH265TranscodingQueue section videos failed", log.Any("sectionId", section.ID), log.E(err))
+				continue
+			}
+			for _, sectionVideo := range sectionVideos {
+				if _, exists := seen[sectionVideo.VideoID]; exists {
+					continue
+				}
+				seen[sectionVideo.VideoID] = struct{}{}
+				videoIDs = append(videoIDs, sectionVideo.VideoID)
+			}
+		}
+	}
+
+	modified, err := vidmod.QueueH265Transcode(videoIDs)
+	if err != nil {
+		log.Warn("ScanHomeH265TranscodingQueue enqueue failed", log.E(err))
+		return
+	}
+	if modified > 0 {
+		log.Info("ScanHomeH265TranscodingQueue queued videos", log.Any("count", modified))
+	}
+}
+
+func submitH265Video(ctx context.Context, domains laosiji.SystemDomainsResp, video *vidmod.VideoModel, maxRunning int) error {
+	if video == nil || strings.TrimSpace(video.SourceURL) == "" {
+		return nil
+	}
+	claimedAt, claimed, err := vidmod.ClaimH265Pending(video.ID)
+	if err != nil || !claimed {
+		return err
+	}
+	video.H265PendingAt = claimedAt
+	result, err := submitH265Transcode(ctx, domains, video, maxRunning)
+	if err != nil {
+		if errors.Is(err, laosiji.ErrTranscodeQueueFull) {
+			marked, markErr := vidmod.MarkH265Queued(video.ID, video.SourceURL, video.H265PendingAt)
+			if markErr != nil {
+				log.Warn("submitH265Video restore queue failed", log.Any("videoId", video.ID), log.E(markErr))
+			} else if marked {
+				upsertH265TaskWithError(video, domains, "", hevctaskmod.H265TaskStatusQueued, err.Error())
+			}
+			return err
+		}
+		marked, markErr := vidmod.MarkH265Failed(video.ID, video.SourceURL, video.H265PendingAt)
+		if markErr != nil {
+			log.Warn("submitH265Video mark failed", log.Any("videoId", video.ID), log.E(markErr))
+		} else if marked {
+			upsertH265TaskWithError(video, domains, "", hevctaskmod.H265TaskStatusFailed, err.Error())
+		}
+		return err
+	}
+	applyH265Result(video, result)
+	return nil
+}
+
+func submitH265Transcode(ctx context.Context, domains laosiji.SystemDomainsResp, video *vidmod.VideoModel, maxRunning int) (laosiji.TranscodeResult, error) {
+	fileURL := hevcAppTranscodeM3u8URL(video.SourceURL, video.H265PendingAt)
+	fileID := h265CloudFileID(video)
+	if fileURL == "" || fileID == "" {
+		return laosiji.TranscodeResult{}, fmt.Errorf("empty signed app transcode m3u8 url, videoId:%s", video.ID.Hex())
+	}
+	return laosiji.SubmitH264ToH265Task(ctx, fileID, fileURL, domains, maxRunning)
+}
+
+func queryH265Transcode(ctx context.Context, domains laosiji.SystemDomainsResp, video *vidmod.VideoModel) (laosiji.TranscodeResult, bool, error) {
+	fileURL := hevcAppTranscodeM3u8URL(video.SourceURL, video.H265PendingAt)
+	fileID := h265CloudFileID(video)
+	if fileURL == "" || fileID == "" {
+		return laosiji.TranscodeResult{}, false, fmt.Errorf("empty signed app transcode m3u8 url, videoId:%s", video.ID.Hex())
+	}
+	return laosiji.QueryH264ToH265Task(ctx, fileID, fileURL, domains)
+}
+
+// hevcAppTranscodeM3u8URL 使用 claim 时间生成稳定、限时且绑定源路径的
+// HMAC 拉流 URL。云端 file_id 独立计算，签名或域名变化不会影响轮询。
+func hevcAppTranscodeM3u8URL(source string, claimedAt time.Time) string {
+	if skdg.Conf == nil {
+		return ""
+	}
+	if claimedAt.IsZero() {
+		return ""
+	}
+	return buildH265AppTranscodeM3u8URL(
+		skdg.Conf.Url.AppApiUrl,
+		source,
+		skdg.Conf.Hevc.PullSecret,
+		claimedAt.UTC().Truncate(time.Second).Add(h265PullURLTTL),
+	)
+}
+
+func buildH265AppTranscodeM3u8URL(baseURL, source, pullSecret string, expiresAt time.Time) string {
+	baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	normalizedSource, sourceErr := hevcpull.NormalizeSource(source)
+	if baseURL == "" || sourceErr != nil {
+		return ""
+	}
+	parsed, err := url.Parse(baseURL)
+	if err != nil ||
+		//parsed.Scheme != "https" ||
+		parsed.Host == "" ||
+		parsed.User != nil ||
+		(parsed.Path != "" && parsed.Path != "/") ||
+		parsed.RawPath != "" ||
+		parsed.RawQuery != "" ||
+		parsed.Fragment != "" {
+		return ""
+	}
+	rawURL := baseURL + "/api/app/vid/transcode/m3u8/" + normalizedSource
+	signedURL, err := hevcpull.SignURL(rawURL, pullSecret, expiresAt)
+	if err != nil {
+		return ""
+	}
+	return signedURL
+}
+
+// h265CloudFileID is independent from the temporary signed fetch URL. It
+// changes when the source or retry attempt changes, but not when App domains,
+// expiry timestamps, or signing keys rotate during an in-flight attempt.
+func h265CloudFileID(video *vidmod.VideoModel) string {
+	if video == nil || video.ID.IsZero() {
+		return ""
+	}
+	source, err := hevcpull.NormalizeSource(video.SourceURL)
+	if err != nil {
+		return ""
+	}
+	identity := fmt.Sprintf(
+		"91porn:h265:v1\x00%s\x00%s\x00%d",
+		video.ID.Hex(),
+		source,
+		video.H265FailCount,
+	)
+	return laosiji.TranscodeFileID(identity)
+}
+
+func applyH265Result(video *vidmod.VideoModel, result laosiji.TranscodeResult) {
+	h265URL := ""
+	taskStatus := hevctaskmod.H265TaskStatusProcessing
+	switch {
+	case result.Done:
+		h265URL = h265PlayableURL(video, result)
+		if h265URL == "" {
+			marked, markErr := vidmod.MarkH265Failed(video.ID, video.SourceURL, video.H265PendingAt)
+			if markErr != nil {
+				log.Warn("applyH265Result mark empty result failed", log.Any("videoId", video.ID), log.E(markErr))
+			} else if marked {
+				_ = hevctaskmod.UpsertWithError(video.ID, video.Title, result.FileURL, result.FileID, "", hevctaskmod.H265TaskStatusFailed, "empty H265 URL in completed task")
+			}
+			return
+		}
+		updated, err := vidmod.MarkPendingH265Success(video.ID, video.SourceURL, video.H265PendingAt, h265URL)
+		if err != nil {
+			log.Warn("applyH265Result MarkPendingH265Success failed", log.Any("videoId", video.ID), log.E(err))
+			return
+		}
+		if !updated {
+			// 当前视频可能已经被老司机回填 H.265，或 sourceURL 已经变化。
+			// 对仍处于 pending 且缺少地址的新源重新排队；已有成功地址时该操作自动跳过。
+			_, _ = vidmod.MarkH265Queued(video.ID, video.SourceURL, video.H265PendingAt)
+			// 不更新 hevc_task：并发轮询的另一个实例可能已经成功落库，
+			// 此处覆盖会把正确的 success 审计状态错误回退成 failed。
+			log.Info("applyH265Result ignored stale result", log.Any("videoId", video.ID))
+			return
+		}
+		clearH265VideoCache(video.ID)
+		taskStatus = hevctaskmod.H265TaskStatusSuccess
+	case result.Failed:
+		marked, markErr := vidmod.MarkH265Failed(video.ID, video.SourceURL, video.H265PendingAt)
+		if markErr != nil {
+			log.Warn("applyH265Result MarkH265Failed failed", log.Any("videoId", video.ID), log.E(markErr))
+			return
+		}
+		if !marked {
+			log.Info("applyH265Result ignored stale failure", log.Any("videoId", video.ID))
+			return
+		}
+		taskStatus = hevctaskmod.H265TaskStatusFailed
+	default:
+		// processing 结果不改变视频状态，也没有可用于跨集合原子写入的
+		// ModifiedCount。保留现有审计记录，避免旧轮询覆盖并发成功结果。
+		return
+	}
+	if err := hevctaskmod.UpsertWithError(video.ID, video.Title, result.FileURL, result.FileID, h265URL, taskStatus, result.ErrorMsg); err != nil {
+		log.Warn("applyH265Result upsert task failed", log.Any("videoId", video.ID), log.E(err))
+	}
+}
+
+// h265PlayableURL 区分云端“无需转码”和真正生成了新 H265 文件的结果。
+// 无需转码时源地址本身已经可直接使用，不能把我方 App 拉流 URL 错存成老司机资源路径。
+func h265PlayableURL(video *vidmod.VideoModel, result laosiji.TranscodeResult) string {
+	if result.NoNeed {
+		if video == nil {
+			return ""
+		}
+		return strings.TrimSpace(video.SourceURL)
+	}
+	return laosiji.MovieM3u8SourcePath(result.HevcURL)
+}
+
+func clearH265VideoCache(videoID primitive.ObjectID) {
+	if skdg.Redis == nil || videoID.IsZero() {
+		return
+	}
+	key := fmt.Sprintf(redisconst.VideoInfoKey(), videoID.Hex())
+	if _, err := skdg.Redis.Del(key); err != nil {
+		log.Warn("clearH265VideoCache failed", log.Any("videoId", videoID), log.E(err))
+	}
+}
+
+func upsertH265TaskWithError(video *vidmod.VideoModel, domains laosiji.SystemDomainsResp, h265URL string, status hevctaskmod.H265TaskStatus, errorMsg string) {
+	if video == nil {
+		return
+	}
+	submitURL, fileID := h265TaskSubmitInfo(video, domains)
+	if err := hevctaskmod.UpsertWithError(video.ID, video.Title, submitURL, fileID, h265URL, status, errorMsg); err != nil {
+		log.Warn("upsertH265TaskWithError failed", log.Any("videoId", video.ID), log.E(err))
+	}
+}
+
+func h265TaskSubmitInfo(video *vidmod.VideoModel, _ laosiji.SystemDomainsResp) (string, string) {
+	if video == nil {
+		return "", ""
+	}
+	submitURL := strings.TrimSpace(hevcAppTranscodeM3u8URL(video.SourceURL, video.H265PendingAt))
+	if submitURL == "" {
+		submitURL = strings.TrimSpace(video.SourceURL)
+	}
+	fileID := h265CloudFileID(video)
+	// 按运维要求持久化“可直接访问”的签名 URL，便于从 hevc_task 复制出来核验。
+	// 代价：库中会保存有效期内可用的签名拉流地址（含 HMAC），务必收紧 DB 访问权限。
+	return submitURL, fileID
+}
+
+func h265MaxRunning() int {
+	cfg, err := sysconfmod.GetByVCode(sysconfmod.VCodeH265MaxRunning)
+	if err != nil {
+		return h265DefaultMaxRunning
+	}
+	maxRunning, err := strconv.Atoi(cfg.Value)
+	if err != nil || maxRunning <= 0 {
+		return h265DefaultMaxRunning
+	}
+	return maxRunning
+}
+
+func h265HomeModuleIDs() ([]primitive.ObjectID, []string) {
+	modules, err := moduleconfmod.GetModuleConfByType(moduleconfmod.HomePage)
+	if err != nil {
+		log.Warn("h265HomeModuleIDs query homepage modules failed", log.E(err))
+		return nil, nil
+	}
+	moduleIDs := make([]primitive.ObjectID, 0, len(modules))
+	moduleIDHexes := make([]string, 0, len(modules))
+	for _, module := range modules {
+		if module.Status != 1 || module.ID.IsZero() {
+			continue
+		}
+		moduleIDs = append(moduleIDs, module.ID)
+		moduleIDHexes = append(moduleIDHexes, module.ID.Hex())
+	}
+	return moduleIDs, moduleIDHexes
+}
+
+func appendUniqueH265VideoIDs(ids *[]primitive.ObjectID, seen map[primitive.ObjectID]struct{}, videos []*vidmod.VideoModel) {
+	for _, video := range videos {
+		if video == nil {
+			continue
+		}
+		if _, exists := seen[video.ID]; exists {
+			continue
+		}
+		seen[video.ID] = struct{}{}
+		*ids = append(*ids, video.ID)
+	}
+}
+
+func homeH265Filter(moduleID string, sortType int) primitive.M {
+	return bson.M{
+		"status":    vidmod.CheckPass,
+		"newsType":  vidmod.SP,
+		"mId":       moduleID,
+		"sourceURL": bson.M{"$exists": true, "$ne": ""},
+		"deleteAt":  bson.M{"$exists": false},
+	}
+}
+
+func homeH265Options(sortType int) *options.FindOptions {
+	var sort bson.D
+	switch sortType {
+	case 2:
+		sort = bson.D{
+			{Key: "liaoBaTopSort", Value: -1},
+			{Key: "likeCount", Value: -1},
+			{Key: "reviewAt", Value: -1},
+		}
+	case 3:
+		sort = bson.D{{Key: "playCount", Value: -1}, {Key: "reviewAt", Value: -1}}
+	case 7:
+		sort = bson.D{{Key: "collectCount", Value: -1}, {Key: "reviewAt", Value: -1}}
+	case 9:
+		sort = bson.D{{Key: "hot", Value: -1}, {Key: "reviewAt", Value: -1}}
+	default:
+		sort = bson.D{{Key: "reviewAt", Value: -1}}
+	}
+	return options.Find().SetLimit(h265HomeCandidateSize).SetSort(sort)
+}

skd/job/hevc_transcode_test.go

@@ -0,0 +1,330 @@
+package job
+
+import (
+	"net/url"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"91porn-server/common/hevcpull"
+	"91porn-server/common/laosiji"
+	"91porn-server/models/v/vidmod"
+	"91porn-server/skd/skdg"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func TestBuildH265AppTranscodeM3u8URL(t *testing.T) {
+	const pullSecret = "test-pull-secret-strong-32-bytes!!"
+	now := time.Date(2026, 7, 25, 12, 0, 0, 0, time.UTC)
+	expiresAt := now.Add(h265PullURLTTL)
+	tests := []struct {
+		name     string
+		base     string
+		source   string
+		wantPath string
+	}{
+		{
+			name:     "normalizes slashes",
+			base:     " https://app.example.com/ ",
+			source:   " /video/demo/index.m3u8 ",
+			wantPath: "/api/app/vid/transcode/m3u8/video/demo/index.m3u8",
+		},
+		{name: "empty base", source: "video.m3u8"},
+		{name: "empty source", base: "https://app.example.com"},
+		{name: "base without scheme", base: "app.example.com", source: "video.m3u8"},
+		{name: "plain http is forbidden", base: "http://app.example.com", source: "video.m3u8"},
+		{name: "unsupported scheme", base: "ftp://app.example.com", source: "video.m3u8"},
+		{name: "base with path", base: "https://app.example.com/prefix", source: "video.m3u8"},
+		{name: "base with query", base: "https://app.example.com?token=x", source: "video.m3u8"},
+		{name: "source traversal", base: "https://app.example.com", source: "video/../index.m3u8"},
+		{name: "absolute source", base: "https://app.example.com", source: "https://cdn.example/index.m3u8"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := buildH265AppTranscodeM3u8URL(tt.base, tt.source, pullSecret, expiresAt)
+			if tt.wantPath == "" {
+				if got != "" {
+					t.Fatalf("buildH265AppTranscodeM3u8URL() = %q, want empty", got)
+				}
+				return
+			}
+			parsed, err := url.Parse(got)
+			if err != nil {
+				t.Fatalf("parse signed URL: %v", err)
+			}
+			if parsed.Path != tt.wantPath {
+				t.Fatalf("signed URL path = %q, want %q", parsed.Path, tt.wantPath)
+			}
+			if err = hevcpull.VerifyURL(parsed, pullSecret, now); err != nil {
+				t.Fatalf("signed URL did not verify: %v", err)
+			}
+		})
+	}
+}
+
+func TestHevcAppTranscodeM3u8URLUsesStableClaimTime(t *testing.T) {
+	oldConf := skdg.Conf
+	t.Cleanup(func() {
+		skdg.Conf = oldConf
+	})
+	claimedAt := time.Date(2026, 7, 25, 12, 0, 0, 0, time.UTC)
+
+	skdg.Conf = nil
+	if got := hevcAppTranscodeM3u8URL("video.m3u8", claimedAt); got != "" {
+		t.Fatalf("nil config returned %q", got)
+	}
+
+	skdg.Conf = &skdg.GlobalConfig{}
+	skdg.Conf.Url.AppApiUrl = "https://app.example.com"
+	skdg.Conf.Hevc.PullSecret = "test-pull-secret-strong-32-bytes!!"
+	first := hevcAppTranscodeM3u8URL("video.m3u8", claimedAt)
+	second := hevcAppTranscodeM3u8URL("video.m3u8", claimedAt)
+	if first == "" || first != second {
+		t.Fatalf("signed URL must be stable for one claim: first=%q second=%q", first, second)
+	}
+	parsed, err := url.Parse(first)
+	if err != nil {
+		t.Fatalf("parse signed URL: %v", err)
+	}
+	if !strings.HasPrefix(parsed.Path, "/api/app/vid/transcode/m3u8/") {
+		t.Fatalf("unexpected signed URL path: %s", parsed.Path)
+	}
+	if err = hevcpull.VerifyURL(parsed, skdg.Conf.Hevc.PullSecret, claimedAt); err != nil {
+		t.Fatalf("signed URL did not verify: %v", err)
+	}
+}
+
+func TestH265CloudFileIDIsIndependentFromSignedPullURL(t *testing.T) {
+	video := &vidmod.VideoModel{
+		ID:            primitive.NewObjectID(),
+		SourceURL:     " /sp/movie/index.m3u8 ",
+		H265FailCount: 1,
+		H265PendingAt: time.Date(2026, 7, 25, 12, 0, 0, 0, time.UTC),
+	}
+	first := h265CloudFileID(video)
+	if first == "" {
+		t.Fatal("empty cloud file ID")
+	}
+
+	video.H265PendingAt = video.H265PendingAt.Add(time.Hour)
+	video.SourceURL = "/sp/movie/index.m3u8"
+	if got := h265CloudFileID(video); got != first {
+		t.Fatalf("equivalent source or claim time changed cloud file ID: %q != %q", got, first)
+	}
+
+	video.SourceURL = "/sp/other/index.m3u8"
+	if got := h265CloudFileID(video); got == first {
+		t.Fatal("different source reused cloud file ID")
+	}
+	video.SourceURL = "/sp/movie/index.m3u8"
+	video.H265FailCount++
+	if got := h265CloudFileID(video); got == first {
+		t.Fatal("new retry attempt reused cloud file ID")
+	}
+}
+
+func TestH265TranscodingJobEnabledRequiresEnvironmentSwitchAndConfig(t *testing.T) {
+	oldConf := skdg.Conf
+	oldLSJConfig := laosiji.Config{
+		AppID:     laosiji.Appid,
+		APIKey:    laosiji.APIKey,
+		APIUrl:    laosiji.APIUrl,
+		ImageYuan: laosiji.IMAGEYUAN,
+		NoticeURL: laosiji.NoticeURL,
+	}
+	t.Cleanup(func() {
+		skdg.Conf = oldConf
+		laosiji.InitConfig(oldLSJConfig)
+	})
+
+	skdg.Conf = &skdg.GlobalConfig{}
+	skdg.Conf.Base.Env = "prod"
+	skdg.Conf.Url.AppApiUrl = "https://app.example.com"
+	laosiji.InitConfig(laosiji.Config{})
+	if h265TranscodingJobEnabled() {
+		t.Fatal("job enabled without laosiji credentials")
+	}
+
+	laosiji.InitConfig(laosiji.Config{
+		AppID:  "test-app",
+		APIKey: "test-key",
+		APIUrl: "https://laosiji.example.com",
+	})
+	skdg.Conf.Base.Env = "test"
+	if h265TranscodingJobEnabled() {
+		t.Fatal("job enabled in test without explicit switch")
+	}
+
+	enabled := true
+	skdg.Conf.Hevc.EnableTranscode = &enabled
+	if h265TranscodingJobEnabled() {
+		t.Fatal("job enabled in test without hevc.pullSecret")
+	}
+
+	skdg.Conf.Hevc.PullSecret = "test-pull-secret-strong-32-bytes!!"
+	skdg.Conf.Url.AppApiUrl = ""
+	if h265TranscodingJobEnabled() {
+		t.Fatal("job enabled without appApiUrl")
+	}
+
+	skdg.Conf.Url.AppApiUrl = "https://app.example.com"
+	if !h265TranscodingJobEnabled() {
+		t.Fatal("job disabled in test with explicit switch and complete configuration")
+	}
+
+	skdg.Conf.Base.Env = "prod"
+	skdg.Conf.Hevc.EnableTranscode = nil
+	if !h265TranscodingJobEnabled() {
+		t.Fatal("job disabled with complete production configuration")
+	}
+
+	disabled := false
+	skdg.Conf.Hevc.EnableTranscode = &disabled
+	if h265TranscodingJobEnabled() {
+		t.Fatal("job enabled after explicit production kill switch")
+	}
+}
+
+func TestH265PlayableURL(t *testing.T) {
+	video := &vidmod.VideoModel{SourceURL: "sp/source/index.m3u8"}
+
+	if got := h265PlayableURL(video, laosiji.TranscodeResult{
+		Done:    true,
+		NoNeed:  true,
+		HevcURL: "https://app.example.com/api/app/vid/h5/light/m3u8/sp/source/index.m3u8",
+	}); got != video.SourceURL {
+		t.Fatalf("no-need result returned %q, want original source %q", got, video.SourceURL)
+	}
+
+	if got := h265PlayableURL(video, laosiji.TranscodeResult{
+		Done:    true,
+		HevcURL: "https://cdn.example.com/m3m/transcoded/index.m3u8",
+	}); got != "laosiji/m3m/transcoded/index.m3u8" {
+		t.Fatalf("transcoded result returned %q", got)
+	}
+
+	if got := h265PlayableURL(nil, laosiji.TranscodeResult{Done: true, NoNeed: true}); got != "" {
+		t.Fatalf("nil video no-need result returned %q", got)
+	}
+}
+
+func TestH265PendingTimedOut(t *testing.T) {
+	now := time.Date(2026, 7, 25, 12, 0, 0, 0, time.UTC)
+	tests := []struct {
+		name  string
+		video *vidmod.VideoModel
+		want  bool
+	}{
+		{name: "nil video"},
+		{
+			name: "not pending",
+			video: &vidmod.VideoModel{
+				H265Status:    vidmod.H265StatusQueued,
+				H265PendingAt: now.Add(-25 * time.Hour),
+			},
+		},
+		{
+			name: "missing claim time",
+			video: &vidmod.VideoModel{
+				H265Status: vidmod.H265StatusPending,
+			},
+		},
+		{
+			name: "still within timeout",
+			video: &vidmod.VideoModel{
+				H265Status:    vidmod.H265StatusPending,
+				H265PendingAt: now.Add(-h265PendingTimeout + time.Second),
+			},
+		},
+		{
+			name: "at timeout",
+			video: &vidmod.VideoModel{
+				H265Status:    vidmod.H265StatusPending,
+				H265PendingAt: now.Add(-h265PendingTimeout),
+			},
+			want: true,
+		},
+		{
+			name: "past timeout",
+			video: &vidmod.VideoModel{
+				H265Status:    vidmod.H265StatusPending,
+				H265PendingAt: now.Add(-h265PendingTimeout - time.Second),
+			},
+			want: true,
+		},
+		{
+			name: "future timestamp",
+			video: &vidmod.VideoModel{
+				H265Status:    vidmod.H265StatusPending,
+				H265PendingAt: now.Add(time.Second),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := h265PendingTimedOut(tt.video, now); got != tt.want {
+				t.Fatalf("h265PendingTimedOut() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestHomeH265QueryMatchesAppHomepageSorts(t *testing.T) {
+	for _, sortType := range []int{1, 2, 3, 7, 9} {
+		filter := homeH265Filter("module-id", sortType)
+		if _, exists := filter["reviewAt"]; exists {
+			t.Fatalf("sort %d unexpectedly limits reviewAt: %#v", sortType, filter["reviewAt"])
+		}
+	}
+
+	tests := []struct {
+		sortType int
+		want     bson.D
+	}{
+		{
+			sortType: 1,
+			want:     bson.D{{Key: "reviewAt", Value: -1}},
+		},
+		{
+			sortType: 2,
+			want: bson.D{
+				{Key: "liaoBaTopSort", Value: -1},
+				{Key: "likeCount", Value: -1},
+				{Key: "reviewAt", Value: -1},
+			},
+		},
+		{
+			sortType: 3,
+			want: bson.D{
+				{Key: "playCount", Value: -1},
+				{Key: "reviewAt", Value: -1},
+			},
+		},
+		{
+			sortType: 7,
+			want: bson.D{
+				{Key: "collectCount", Value: -1},
+				{Key: "reviewAt", Value: -1},
+			},
+		},
+		{
+			sortType: 9,
+			want: bson.D{
+				{Key: "hot", Value: -1},
+				{Key: "reviewAt", Value: -1},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		got := homeH265Options(tt.sortType).Sort
+		if !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("sort %d = %#v, want %#v", tt.sortType, got, tt.want)
+		}
+	}
+}

skd/job/hotvalue/video.go

@@ -0,0 +1,125 @@
+package hotvalue
+
+import (
+	"91porn-server/common/log"
+	"91porn-server/models/v/cmtmod"
+	"91porn-server/models/v/mediamod"
+	"91porn-server/models/v/vidmod"
+	"github.com/golang-module/carbon/v2"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"math"
+	"time"
+)
+
+type updateFn func(models []mongo.WriteModel) error
+
+var (
+	actions = map[string]updateFn{
+		cmtmod.OTypeVideo:   vidmod.Bulk,
+		cmtmod.OTypeCartoon: mediamod.Bulk,
+	}
+)
+
+// ResetHot 重置热度值
+func ResetHot() {
+	start := carbon.NewCarbon().SubDays(30).StdTime()
+	end := time.Now()
+	writes := make([]mongo.WriteModel, 0)
+
+	for objType, fn := range actions {
+		skip := 0
+		limit := 100
+
+		for {
+			// 获取30天内有评论的objID
+			objIds, hasNext, err := cmtmod.GetObjIdsBetweenTime(objType, start, end, skip, limit)
+			if err != nil {
+				log.Error("Job ResetHot cmtmod.GetObjIdsBetweenTime fail", log.E(err))
+				return
+			}
+
+			for _, objId := range objIds {
+				hotData := resetObjectHot(objId, objType, start, end)
+				if len(hotData) == 0 {
+					continue
+				}
+				update := mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": objId}).SetUpdate(hotData).SetUpsert(true)
+				writes = append(writes, update)
+			}
+
+			if len(writes) > 0 {
+				err = fn(writes)
+				if err != nil {
+					log.Error("Job ResetHot fail", log.E(err),
+						log.Any("objType", objType),
+						log.Any("skip", skip))
+				}
+			}
+			writes = writes[0:0]
+			skip = skip + limit
+
+			if !hasNext {
+				break
+			}
+		}
+	}
+}
+
+// 热度=Z历史所有评论数 × 30% × e^（−{评论天数}×0.1）
+//+Z近7天评论数 × 70%
+
+func resetObjectHot(objId primitive.ObjectID, objType string, start, end time.Time) bson.M {
+	// 暂时只支持视频和ACG的评论
+	if objType != cmtmod.OTypeVideo && objType != cmtmod.OTypeCartoon {
+		return nil
+	}
+	// 获取该对性一段时间内，按天分组，每天的评论总数
+	list, err := cmtmod.GetCountGroupByDay(objId, start, end)
+	if err != nil {
+		log.Error("Job ResetHot resetObjectHot cmtmod.GetCountGroupByDay fail", log.E(err))
+		return nil
+	}
+	var historyHot float64
+	var sevenDayCount float64
+	// 计算出该对象的热度值
+	for _, v := range list {
+		// 获取跟今天相差的天数
+		day := float64(carbon.Now().DiffAbsInDays(carbon.ParseByFormat(v.Time, "Y-m-d")))
+		// 计算历史评论热度
+		hot := calculateCommentScore(v.Count, day)
+		historyHot = historyHot + hot
+		// 累加近七天的总数
+		if day <= 7 {
+			sevenDayCount = sevenDayCount + float64(v.Count)
+		}
+	}
+	currentHot := historyHot + sevenDayCount*0.7
+	log.Info("Job ResetHot resetObjectHot result", log.Any("hot", currentHot))
+
+	data := bson.M{"hot": currentHot}
+
+	if objType == cmtmod.OTypeVideo {
+		data["updatedAt"] = time.Now()
+	} else {
+		data["updateTime"] = time.Now()
+	}
+
+	return bson.M{"$set": data}
+}
+
+func calculateCommentScore(totalComments int64, daysSinceComment float64) float64 {
+	// Ensure we don't have negative days
+	if daysSinceComment < 0 {
+		daysSinceComment = 0
+	}
+
+	// Calculate the decay factor: e^(-days * 0.1)
+	decayFactor := math.Exp(-daysSinceComment * 0.1)
+
+	// Calculate the final score: totalComments * 0.3 * decayFactor
+	score := float64(totalComments) * 0.3 * decayFactor
+
+	return score
+}

skd/job/im_ad_notify.go

@@ -0,0 +1,150 @@
+package job
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	"91porn-server/common/enum/imad"
+	"91porn-server/common/log"
+	"91porn-server/common/stderr"
+	"91porn-server/models/cache/sysconfdata"
+	"91porn-server/models/v/sysconfmod"
+	"91porn-server/skd/skdg"
+	"91porn-server/services/srv_im"
+)
+
+const (
+	imAdNotifyLockKey    = "skd:im_ad_notify:online:lock"
+	imAdNotifyLastRunKey = "skd:im_ad_notify:online:last_run"
+	imAdNotifyLockTTL    = 10 * time.Minute
+)
+
+type imAdNotifyConfig struct {
+	Enable          bool
+	IntervalMinutes int64
+	Positions       []string
+	MaxUsers        int
+}
+
+// IMAdNotifyOnline 按 sysconf.imConfig 定时向当前 IM 在线用户透传广告刷新通知。
+func IMAdNotifyOnline() {
+	if err := imAdNotifyOnline(); err != nil {
+		log.Error("IMAdNotifyOnline failed", log.E(err))
+	}
+}
+
+func imAdNotifyOnline() error {
+	conf := loadIMAdNotifyConfig()
+	now := time.Now()
+	lastRunAt, err := loadIMAdNotifyLastRunAt()
+	if err != nil {
+		return err
+	}
+	if !shouldRunIMAdNotify(conf, lastRunAt, now) {
+		return nil
+	}
+
+	acquired, err := skdg.Redis.Setnx_NewOK(imAdNotifyLockKey, 1, imAdNotifyLockTTL)
+	if err != nil {
+		return err
+	}
+	if !acquired {
+		log.Info("IMAdNotifyOnline skipped: lock held")
+		return nil
+	}
+	defer func() {
+		if _, err := skdg.Redis.Del(imAdNotifyLockKey); err != nil {
+			log.Warn("IMAdNotifyOnline release lock failed", log.E(err))
+		}
+	}()
+
+	lastRunAt, err = loadIMAdNotifyLastRunAt()
+	if err != nil {
+		return err
+	}
+	if !shouldRunIMAdNotify(conf, lastRunAt, now) {
+		return nil
+	}
+
+	resp, code, msg := srv_im.SendAdNotify(srv_im.AdNotifyReq{
+		Target:    "online",
+		Positions: conf.Positions,
+		MaxUsers:  conf.MaxUsers,
+	})
+	if code != stderr.Success {
+		return fmt.Errorf("im ad notify failed: code=%d msg=%s", code, msg)
+	}
+	log.Info("IMAdNotifyOnline done", log.Any("resp", resp))
+	return saveIMAdNotifyLastRunAt(now)
+}
+
+func loadIMAdNotifyConfig() imAdNotifyConfig {
+	conf := imAdNotifyConfig{IntervalMinutes: 60, MaxUsers: 1000}
+	all, err := sysconfdata.GetAllFromCache()
+	if err != nil {
+		log.Warn("load im sysconf failed", log.E(err))
+		return conf
+	}
+	conf.Enable = all.GetBool(sysconfmod.VCodeIMAdEnable)
+	if v := all.GetInt(sysconfmod.VCodeIMAdIntervalMinutes); v > 0 {
+		conf.IntervalMinutes = v
+	}
+	if v := all.GetInt(sysconfmod.VCodeIMAdMaxUsers); v > 0 {
+		conf.MaxUsers = int(v)
+	}
+	// 每个广告位独立开关，开了就加进去
+	conf.Positions = collectEnabledIMAdPositions(all)
+	return conf
+}
+
+// collectEnabledIMAdPositions 把启用了的广告位 code 收集成列表
+func collectEnabledIMAdPositions(all sysconfmod.ConfMap) []string {
+	positions := make([]string, 0, 4)
+	if all.GetBool(sysconfmod.VCodeIMAdPosChatNotificationBar) {
+		positions = append(positions, imad.ChatNotificationBar)
+	}
+	if all.GetBool(sysconfmod.VCodeIMAdPosChatFloatingGifBall) {
+		positions = append(positions, imad.ChatFloatingGifBall)
+	}
+	if all.GetBool(sysconfmod.VCodeIMAdPosListFeedNative) {
+		positions = append(positions, imad.ListFeedNative)
+	}
+	if all.GetBool(sysconfmod.VCodeIMAdPosListBanner) {
+		positions = append(positions, imad.ListBanner)
+	}
+	return positions
+}
+
+func shouldRunIMAdNotify(conf imAdNotifyConfig, lastRunAt time.Time, now time.Time) bool {
+	if !conf.Enable || len(conf.Positions) == 0 {
+		return false
+	}
+	if lastRunAt.IsZero() {
+		return true
+	}
+	return !now.Before(lastRunAt.Add(normalizeIMAdNotifyInterval(conf.IntervalMinutes)))
+}
+
+func normalizeIMAdNotifyInterval(intervalMinutes int64) time.Duration {
+	if intervalMinutes <= 0 {
+		intervalMinutes = 60
+	}
+	return time.Duration(intervalMinutes) * time.Minute
+}
+
+func loadIMAdNotifyLastRunAt() (time.Time, error) {
+	val, err := skdg.Redis.Get(imAdNotifyLastRunKey)
+	if err != nil || val == nil || *val == "" {
+		return time.Time{}, err
+	}
+	unixSec, err := strconv.ParseInt(*val, 10, 64)
+	if err != nil || unixSec <= 0 {
+		return time.Time{}, nil
+	}
+	return time.Unix(unixSec, 0), nil
+}
+
+func saveIMAdNotifyLastRunAt(now time.Time) error {
+	return skdg.Redis.Set(imAdNotifyLastRunKey, strconv.FormatInt(now.Unix(), 10), 0)
+}

skd/job/job.go

@@ -0,0 +1,162 @@
+package job
+
+import (
+	"91porn-server/common"
+	"91porn-server/common/aiService"
+	"91porn-server/common/log"
+	mediaService "91porn-server/common/media"
+	"91porn-server/common/mediaCenter"
+	"91porn-server/models/commod"
+	"91porn-server/skd/job/checkin"
+	"91porn-server/skd/job/dailyAdverCalc"
+	"91porn-server/skd/job/dataclean"
+	"91porn-server/skd/job/hotvalue"
+	"91porn-server/skd/job/shortrecommend"
+	"91porn-server/skd/job/user"
+	"91porn-server/skd/service"
+	"91porn-server/skd/service/contentreviewser"
+	"91porn-server/skd/service/export_task"
+	"91porn-server/skd/skdg"
+	"context"
+	"time"
+)
+
+// start 定时任务 开启
+func start() {
+	_, _ = statCron.AddFunc("0 0 0 * * ?", func() { //每天0点执行一次
+		// 重置预售卡预付权益
+		user.ResetLimit()
+	})
+	if skdg.Conf.ShortRecommendEnabled() {
+		// 定时检查队列完整性；当天健康时只做一次Redis脚本，异常时自动重建。
+		if _, err := statCron.AddFunc(skdg.Conf.ShortRecommendCron(), shortrecommend.Run); err != nil {
+			log.Error("register short recommend cron failed",
+				log.E(err), log.Any("cron", skdg.Conf.ShortRecommendCron()))
+		}
+		// 启动时补建当日队列；分布式锁保证多实例只执行一次。
+		common.Go(shortrecommend.Run)
+	}
+	_, _ = statCron.AddFunc("0 0/5 * * * ?", func() { //每5分钟执行一次统计任务
+		runStat()
+	})
+	_, _ = statCron.AddFunc("@every 5m", func() { //每5分钟执行一次数据同步任务
+		runSync()
+	})
+	_, _ = statCron.AddFunc("0 0/5 * * * ?", func() { //每5分钟执行一次sql折叠任务
+		runSyncSqlcollaspe()
+	})
+	// _, _ = statCron.AddFunc("0 0/5 * * * ?", func() { //每5分钟刷新一下排行榜
+	// 	runRankStat()
+	// })
+	// _, _ = statCron.AddFunc("0 0/1 * * * ?", func() { //每1分钟执行一次统计任务
+	// 	runPromoteStat()
+	// })
+	// _, _ = statCron.AddFunc("0 0 0/1 * * ?", func() { //每1个小时增加内部人员推广人数
+	// 	runInnerFakePromote()
+	// })
+	// _, _ = statCron.AddFunc("0 0/2 * * * ?", func() { //每2分钟生成备用推广码
+	// 	runMakePromotionCode()
+	// })
+	// _, _ = statCron.AddFunc("0 0 0,1,3,6,7,9,10,12,15,20,21,23 * * ?", func() { //每天11，15，18，23点执行一次消息任务
+	// 	service.GoldEggLuckyRunSync()
+	// })
+	_, _ = statCron.AddFunc("0 0 4 * * ?", func() { //每天凌晨4点执行一次
+		dataclean.RunSync()
+	})
+
+	_, _ = statCron.AddFunc("0/10 * * * * ?", func() { //每10秒执行一次
+		service.VidTimeOnline()
+	})
+
+	common.Go(func() {
+		hotvalue.ResetHot()
+	})
+	_, _ = statCron.AddFunc("0 0 6 * * ?", func() { //每天凌晨6点执行一次
+		hotvalue.ResetHot()
+	})
+
+	// 每10秒检测需要导出的任务列表
+	_, _ = statCron.AddFunc("0/10 * * * * ?", func() { //每10秒执行一次
+		export_task.ExecExportTask()
+	})
+	// 每3秒检测有无待执行的内容审查任务（同一时刻仅 1 个任务运行）
+	_, _ = statCron.AddFunc("0/3 * * * * ?", func() {
+		contentreviewser.TryRunPendingTask()
+	})
+
+	_, _ = statCron.AddFunc("0 * * * * ?", func() { // 每分钟检查是否需要发送 IM 广告通知
+		IMAdNotifyOnline()
+	})
+
+	// 每 5 秒检查 web 端是否发起了 IM 用户同步请求
+	_, _ = statCron.AddFunc("0/5 * * * * ?", func() {
+		RunPendingSyncIMUsers()
+	})
+	// 同步acg订阅的动漫
+	_, _ = statCron.AddFunc("0 0/30 * * * ?", func() { //每30分钟执行一次
+		ms := mediaService.NewMediaSync(skdg.Redis, mediaCenter.NewMediaCenterService(mediaCenter.Config{
+			Url:    skdg.Conf.MediaResourcesCfg.UploadMediaUrl,
+			ProId:  skdg.Conf.MediaResourcesCfg.ProId,
+			Secret: skdg.Conf.MediaResourcesCfg.AppSecret,
+		}))
+		ms.SyncAll("系统")
+	})
+	common.Go(func() {
+		err := aiService.NewAiService(
+			aiService.AppId(int(commod.KFK_APPID)),
+			aiService.Url(skdg.Conf.Url.AiServer),
+			aiService.Redis(skdg.Redis),
+		).Sync()
+		if err != nil {
+			log.Error("Sync AI Template fail")
+		}
+	})
+	// 定期同步ai服务里的模版
+	_, _ = statCron.AddFunc("0 0/10 * * * ?", func() { //每10分钟执行一次
+		err := aiService.NewAiService(
+			aiService.AppId(int(commod.KFK_APPID)),
+			aiService.Url(skdg.Conf.Url.AiServer),
+			aiService.Redis(skdg.Redis),
+		).Sync()
+		if err != nil {
+			log.Error("Sync AI Template fail")
+		}
+	})
+	_, _ = statCron.AddFunc("0/2 * * * * ?", func() { //每2秒检查签到奖励视频切片状态
+		checkin.VideoCheckRS()
+	})
+	_, _ = statCron.AddFunc("0 0/1 * * * ?", func() { // 每分钟提交 H265 等待队列
+		SubmitH265TranscodingQueue()
+	})
+	_, _ = statCron.AddFunc("0 0/1 * * * ?", func() { // 每分钟轮询 H265 转码结果
+		CheckH265TranscodingResults()
+	})
+	_, _ = statCron.AddFunc("0 0 0/1 * * ?", func() { // 每小时保障首页重点视频的 H265 资源
+		ScanHomeH265TranscodingQueue()
+	})
+	common.Go(func() {
+		ScanHomeH265TranscodingQueue()
+		SubmitH265TranscodingQueue()
+		CheckH265TranscodingResults()
+	})
+	spec := "0 0/30 * * * ?"
+	if skdg.Conf.Base.Env == "test" {
+		spec = "0 0/1 * * * ?" // 测试环境3分钟刷新一次
+	}
+
+	_, _ = statCron.AddFunc(spec, func() { //每30分钟执行一次
+		now := time.Now()
+		if now.Hour() == 0 || now.Hour() == 1 || now.Hour() == 2 {
+			yesterday := now.AddDate(0, 0, -1)
+			if err := dailyAdverCalc.CalcRetention(context.Background(), yesterday); err != nil {
+				log.Error("CalcRetention failed", log.Any("error", err))
+			}
+		}
+		// 当天的数据，每小时计算一次，确保及时更新看板
+		if err := dailyAdverCalc.CalcRetention(context.Background(), now); err != nil {
+			log.Error("CalcRetention failed", log.Any("error", err))
+		}
+
+	})
+	statCron.Start()
+}

skd/job/schedule.go

@@ -0,0 +1,34 @@
+package job
+
+import (
+	"context"
+	"time"
+
+	"91porn-server/common/log"
+
+	"github.com/robfig/cron/v3"
+)
+
+var statCron *cron.Cron
+
+func init() {
+	statCron = cron.New(cron.WithSeconds()) //从秒开始的定时任务
+}
+
+// 开启定时任务
+func Start() {
+	log.Info("start job...", log.Any("beginAt", time.Now()))
+	start()
+}
+
+func Stop() {
+	ctx := statCron.Stop()
+	<-ctx.Done()
+	err := ctx.Err()
+	//statCron.Stop()后，若job处于静默可能立即退出，再次执行<-ctx.Done()产生Canceled错误，此处忽略该错误
+	if err != nil && err != context.Canceled {
+		log.Error("Skd Job Stop error", log.E(err))
+		return
+	}
+	log.Info("end job", log.Any("endAt", time.Now()))
+}

skd/job/shortrecommend/job.go

@@ -0,0 +1,231 @@
+package shortrecommend
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	redisutil "91porn-server/common/redis"
+	recommendqueue "91porn-server/common/shortrecommend"
+	"91porn-server/models/v/moduleconfmod"
+	"91porn-server/models/v/vidmod"
+	"91porn-server/skd/skdg"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+const (
+	buildLockTTL           = 10 * time.Minute
+	buildLockRenewInterval = 3 * time.Minute
+	buildTimeout           = 30 * time.Minute
+	healthCheckTimeout     = 5 * time.Second
+)
+
+const renewBuildLockScriptSource = `
+if redis.call('GET', KEYS[1]) ~= ARGV[1] then
+  return 0
+end
+return redis.call('EXPIRE', KEYS[1], ARGV[2])
+`
+
+const releaseBuildLockScriptSource = `
+if redis.call('GET', KEYS[1]) ~= ARGV[1] then
+  return 0
+end
+return redis.call('DEL', KEYS[1])
+`
+
+var (
+	renewBuildLockScript   = redisutil.NewScript(renewBuildLockScriptSource)
+	releaseBuildLockScript = redisutil.NewScript(releaseBuildLockScriptSource)
+)
+
+func evalLockScript(
+	ctx context.Context,
+	script *redisutil.Script,
+	key, token string,
+	args ...interface{},
+) (bool, error) {
+	values := append([]interface{}{token}, args...)
+	raw, err := skdg.Redis.RunScriptContext(ctx, script, []string{key}, values...)
+	if err != nil {
+		return false, err
+	}
+	return fmt.Sprint(raw) == "1", nil
+}
+
+func renewBuildLock(ctx context.Context, key, token string) (bool, error) {
+	return evalLockScript(
+		ctx,
+		renewBuildLockScript,
+		key,
+		token,
+		int64(buildLockTTL/time.Second),
+	)
+}
+
+func releaseBuildLock(ctx context.Context, key, token string) error {
+	_, err := evalLockScript(ctx, releaseBuildLockScript, key, token)
+	return err
+}
+
+func keepBuildLock(ctx context.Context, cancel context.CancelFunc, key, token, version string) {
+	ticker := time.NewTicker(buildLockRenewInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			renewed, err := renewBuildLock(ctx, key, token)
+			if err != nil || !renewed {
+				log.Error("short recommend renew build lock failed",
+					log.E(err), log.Any("version", version))
+				cancel()
+				return
+			}
+		}
+	}
+}
+
+func buildQueueVersion(now time.Time, buildToken string) string {
+	return recommendqueue.VersionWithRevision(now, buildToken)
+}
+
+// Run 刷新真实互动分并发布当日全局短视频队列。
+func Run() {
+	now := time.Now()
+	dateVersion := recommendqueue.VersionAt(now)
+	healthCtx, healthCancel := context.WithTimeout(context.Background(), healthCheckTimeout)
+	health, err := recommendqueue.HealthContext(healthCtx, skdg.Redis, dateVersion)
+	healthCancel()
+	if err != nil {
+		log.Error("short recommend queue health check failed",
+			log.E(err), log.Any("dateVersion", dateVersion))
+		return
+	}
+	if health.Healthy {
+		return
+	}
+	log.Warn("short recommend queue unhealthy, start rebuild",
+		log.Any("dateVersion", dateVersion),
+		log.Any("currentVersion", health.Version),
+		log.Any("healthStatus", health.Status),
+		log.Any("queueLength", health.QueueLength),
+		log.Any("metadataLength", health.MetadataLength),
+		log.Any("queueTTLSeconds", int64(health.QueueTTL/time.Second)),
+		log.Any("metadataTTLSeconds", int64(health.MetadataTTL/time.Second)))
+	lockKey := recommendqueue.LockKey(dateVersion)
+	lockToken := primitive.NewObjectID().Hex()
+	lockCtx, lockCancel := context.WithTimeout(context.Background(), healthCheckTimeout)
+	locked, err := skdg.Redis.SetNXContext(lockCtx, lockKey, lockToken, buildLockTTL)
+	lockCancel()
+	if err != nil {
+		log.Error("short recommend acquire build lock failed",
+			log.E(err), log.Any("dateVersion", dateVersion))
+		return
+	}
+	if !locked {
+		return
+	}
+	buildCtx, cancel := context.WithTimeout(context.Background(), buildTimeout)
+	lockKeeperDone := make(chan struct{})
+	go func() {
+		defer close(lockKeeperDone)
+		keepBuildLock(buildCtx, cancel, lockKey, lockToken, dateVersion)
+	}()
+	defer func() {
+		cancel()
+		<-lockKeeperDone
+		releaseCtx, releaseCancel := context.WithTimeout(context.Background(), healthCheckTimeout)
+		defer releaseCancel()
+		if err := releaseBuildLock(releaseCtx, lockKey, lockToken); err != nil {
+			log.Error("short recommend release build lock failed",
+				log.E(err), log.Any("dateVersion", dateVersion))
+		}
+	}()
+
+	// 首次健康检查与加锁之间可能已有其他实例完成发布；持锁后复检避免重复全量扫描。
+	recheckCtx, recheckCancel := context.WithTimeout(buildCtx, healthCheckTimeout)
+	rechecked, recheckErr := recommendqueue.HealthContext(recheckCtx, skdg.Redis, dateVersion)
+	recheckCancel()
+	if recheckErr != nil {
+		log.Error("short recommend queue health recheck failed",
+			log.E(recheckErr), log.Any("dateVersion", dateVersion))
+		return
+	}
+	if rechecked.Healthy {
+		return
+	}
+
+	queueVersion := buildQueueVersion(now, lockToken)
+	buildStartedAt := time.Now()
+	excludedModuleIDs, err := moduleconfmod.ExcludedVideoModuleIDs(now, true)
+	if err != nil {
+		log.Error("short recommend load excluded modules failed",
+			log.E(err), log.Any("version", queueVersion))
+		return
+	}
+	refreshStartedAt := time.Now()
+	initializationLimit := skdg.Conf.ShortRecommendInitializationLimit()
+	candidates, err := vidmod.RefreshRecommendScoresWithInitializationLimit(
+		buildCtx, now, initializationLimit, excludedModuleIDs...,
+	)
+	refreshDuration := time.Since(refreshStartedAt)
+	if err != nil {
+		log.Error("short recommend refresh scores failed",
+			log.E(err), log.Any("version", queueVersion),
+			log.Any("initializationWriteLimit", initializationLimit),
+			log.Any("refreshDurationMs", refreshDuration.Milliseconds()))
+		return
+	}
+	assembleStartedAt := time.Now()
+	ids, assemblyStats := recommendqueue.AssembleWithStats(candidates, now)
+	assembleDuration := time.Since(assembleStartedAt)
+	if len(ids) == 0 {
+		log.Warn("short recommend candidates empty, keep previous queue",
+			log.Any("version", queueVersion))
+		return
+	}
+	if err := buildCtx.Err(); err != nil {
+		log.Error("short recommend build lock lost before publish",
+			log.E(err), log.Any("version", queueVersion))
+		return
+	}
+	if currentVersion := recommendqueue.VersionAt(time.Now()); currentVersion != dateVersion {
+		log.Warn("short recommend build crossed version boundary, skip publish",
+			log.Any("version", queueVersion), log.Any("currentDateVersion", currentVersion))
+		return
+	}
+	renewed, err := renewBuildLock(buildCtx, lockKey, lockToken)
+	if err != nil || !renewed {
+		log.Error("short recommend verify build lock failed",
+			log.E(err), log.Any("version", queueVersion))
+		return
+	}
+	publishStartedAt := time.Now()
+	err = recommendqueue.Publish(buildCtx, skdg.Redis, recommendqueue.QueueMeta{
+		Version: queueVersion, Length: len(ids), GeneratedAt: now, StartOffset: 0,
+	}, ids, lockKey, lockToken)
+	publishDuration := time.Since(publishStartedAt)
+	if err != nil {
+		log.Error("short recommend publish queue failed",
+			log.E(err), log.Any("version", queueVersion),
+			log.Any("publishDurationMs", publishDuration.Milliseconds()))
+		return
+	}
+	log.Info("short recommend queue published",
+		log.Any("version", queueVersion),
+		log.Any("dateVersion", dateVersion),
+		log.Any("candidateCount", len(candidates)),
+		log.Any("excludedModuleCount", len(excludedModuleIDs)),
+		log.Any("queueLength", len(ids)),
+		log.Any("highScoreCount", assemblyStats.HighCount),
+		log.Any("newVideoCount", assemblyStats.NewCount),
+		log.Any("blockCount", assemblyStats.BlockCount),
+		log.Any("refreshDurationMs", refreshDuration.Milliseconds()),
+		log.Any("assembleDurationMs", assembleDuration.Milliseconds()),
+		log.Any("publishDurationMs", publishDuration.Milliseconds()),
+		log.Any("totalDurationMs", time.Since(buildStartedAt).Milliseconds()))
+}

skd/job/shortrecommend/job_test.go

@@ -0,0 +1,18 @@
+package shortrecommend
+
+import (
+	"testing"
+	"time"
+)
+
+func TestBuildQueueVersionIsUniqueWithinSameDay(t *testing.T) {
+	now := time.Date(2026, 7, 31, 2, 0, 0, 0, time.UTC)
+	first := buildQueueVersion(now, "build-a")
+	second := buildQueueVersion(now, "build-b")
+	if first != "20260731-build-a" || second != "20260731-build-b" {
+		t.Fatalf("versions = %q, %q", first, second)
+	}
+	if first == second {
+		t.Fatal("same-day rebuilds must not reuse a queue version")
+	}
+}

skd/job/sqlcollapser/shordvideo/method.go

@@ -0,0 +1,27 @@
+package shortvideo
+
+import (
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/log"
+	"91porn-server/common/redis"
+	"91porn-server/models/v/vidmod"
+)
+
+// SyncDataToRedis 同步数据到redis
+func SyncDataToRedis(data []*vidmod.VideoModel) error {
+	if len(data) == 0 {
+		return nil
+	}
+
+	for _, v := range data {
+		if v.NewsType != vidmod.SHORT || (v.Status != vidmod.CheckPass && v.Status != vidmod.Free) {
+			// 去除
+			_, _ = redis.Handler.SRem(redisconst.ShortVideosKey, v.ID.Hex())
+		} else {
+			_, _ = redis.Handler.SAdd(redisconst.ShortVideosKey, v.ID.Hex())
+		}
+	}
+
+	log.Info("Redis sync video data completed.")
+	return nil
+}

skd/job/sqlcollapser/shordvideo/sync.go

@@ -0,0 +1,89 @@
+package shortvideo
+
+import (
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/redis"
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/v/vidmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("Redis short video sync start...")
+	defer log.Info("Redis short video sync end.")
+	// 如果短视频缓存存在的数据量过低，表示缓存被清理过，需要重新同步
+	c, err := redis.Handler.SCard(redisconst.ShortVideosKey)
+	if err == nil && c < 1000 {
+		_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, defaultDate)
+	}
+
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.RedisSyncShortVideoJob, defaultDate)
+	if err != nil {
+		log.Error("Redis sync short video data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now()
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.OneDayScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.OneDayScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+
+		//// 获取MID
+		//ret, err := moduleconfmod.GetModuleConfByType(moduleconfmod.ShortPage)
+		//if err != nil {
+		//	log.Error("Redis sync short video data RunSync get module conf failed", log.E(err))
+		//	return
+		//}
+		//midMap := make(map[string]any)
+		//for _, v := range ret {
+		//	midMap[v.ID.Hex()] = nil
+		//}
+
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		var page, size = 1, 1000
+		hasNext := true
+		var data []*vidmod.VideoModel
+		for hasNext {
+			data, hasNext, err = vidmod.GetShortVideoListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
+			if err != nil {
+				log.Error("Redis sync short video data GetVideoListByUpdateTimeRange err! ", log.E(err))
+				return
+			}
+			if hasNext && len(data) > 0 {
+				_ = SyncDataToRedis(data)
+				page++
+			} else {
+				if err = commit(data, subTimeRange.Tail); err != nil {
+					log.Error("Redis sync short video data RunSync commit err! ", log.E(err))
+					return
+				}
+			}
+		}
+
+		lastSyncAt = end
+	}
+}
+
+func commit(data []*vidmod.VideoModel, syncTime time.Time) error {
+	if len(data) <= 0 {
+		_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime)
+		return nil
+	}
+	if err := SyncDataToRedis(data); err != nil {
+		return fmt.Errorf("Redis sync commit short video data SyncDataToES failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime); err != nil {
+		return fmt.Errorf("Redis sync commit short video data UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}

skd/job/sqlcollapser/user/method.go

@@ -0,0 +1,44 @@
+package user
+
+import (
+	"91porn-server/models/v/usermod"
+	"91porn-server/models/v/vidmod"
+	"time"
+)
+
+func doSyncWorkCount(start, end time.Time) error {
+
+	mats := []vidmod.Matcher{
+		(&vidmod.StatusMatch{Status: vidmod.CheckPass}).New(),
+		(&vidmod.UpdatedAtGTEAndLTMatch{GTE: &start, LT: &end}).New(),
+	}
+	publisherIDs, err := vidmod.PublisherIDList(mats...)
+	if err != nil {
+		return err
+	}
+	// 分批操作
+	for len(publisherIDs) > 0 {
+		var uids []uint64
+		if len(publisherIDs) > 100 {
+			uids = publisherIDs[:100]
+			publisherIDs = publisherIDs[100:]
+		} else {
+			uids = publisherIDs
+			publisherIDs = nil
+		}
+
+		mats = []vidmod.Matcher{
+			(&vidmod.StatusMatch{Status: vidmod.CheckPass}).New(),
+			(&vidmod.PublisherIDInMatch{PublisherIDList: uids}).New(),
+		}
+		m, err := vidmod.CountMapByPublisherID(mats...)
+		if err != nil {
+			return err
+		}
+		err = usermod.UpdateUserCountWorks(m)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

skd/job/sqlcollapser/user/runSync.go

@@ -0,0 +1,54 @@
+package user
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("SqlCollasper sync")
+	defer log.Info("SqlCollasper sync user total work  end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SqlCollasperUserTotalWork, defaultDate)
+	if err != nil {
+		log.Error("SqlCollasper RunSync user total work get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now()
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end time.Time
+		if lastSyncAt.Add(time.Hour * 24).Before(now) {
+			end = lastSyncAt.Add(time.Hour * 24)
+		} else {
+			end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+			if end.After(recentMinute) {
+				end = recentMinute
+			}
+		}
+
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		if err = commit(subTimeRange); err != nil {
+			log.Error("SqlCollasper sync audiobook data  RunSync commit err! ", log.E(err))
+			return
+		}
+		lastSyncAt = end
+	}
+}
+
+func commit(t timerange.TimeRange) error {
+	if err := doSyncWorkCount(t.Head, t.Tail); err != nil {
+		return fmt.Errorf("SqlCollasper sync commit incPageView failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SqlCollasperUserTotalWork, t.Tail); err != nil {
+		return fmt.Errorf("SqlCollasper sync commit  UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}

skd/job/sqlcollapser/vid/method.go

@@ -0,0 +1,51 @@
+package vid
+
+import (
+	"time"
+
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/v/vidmod"
+	"91porn-server/skd/skdg"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+func incPageView(syncTime time.Time) error {
+	now := syncTime
+	recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	key := redisconst.VidPageViewListKey(recentMinute.Unix())
+	count, err := skdg.Redis.LCount(key)
+	if err != nil {
+		return err
+	}
+	if count == 0 {
+		return nil
+	}
+	strList, err := skdg.Redis.LRange(key, 0, count-1)
+	if err != nil {
+		return err
+	}
+	dmap := make(map[string]int64)
+	for i := range strList {
+		dmap[strList[i]]++
+	}
+	models := make([]mongo.WriteModel, 0, len(dmap))
+	for k, v := range dmap {
+		_id, err1 := primitive.ObjectIDFromHex(k)
+		if err1 != nil {
+			continue
+		}
+		filter := bson.M{"_id": _id}
+		update := bson.M{"$inc": bson.M{"pageViewCount": v}}
+		models = append(models, mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update))
+	}
+	if err = vidmod.Bulk(models); err != nil {
+		return err
+	}
+	_, _ = skdg.Redis.Del(key)
+	return nil
+}

skd/job/sqlcollapser/vid/runSync.go

@@ -0,0 +1,48 @@
+package vid
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("SqlCollasper sync start...")
+	defer log.Info("SqlCollasper sync end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SqlCollasperVidPageView, defaultDate)
+	if err != nil {
+		log.Error("SqlCollasper RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now().Add(-time.Minute * 5)
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		if err = commit(subTimeRange.Tail); err != nil {
+			log.Error("SqlCollasper sync audiobook data RunSync commit err! ", log.E(err))
+			return
+		}
+		lastSyncAt = end
+	}
+}
+
+func commit(syncTime time.Time) error {
+	if err := incPageView(syncTime); err != nil {
+		return fmt.Errorf("SqlCollasper sync commit incPageView failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SqlCollasperVidPageView, syncTime); err != nil {
+		return fmt.Errorf("SqlCollasper sync commit  UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}

skd/job/sqlcollasper.go

@@ -0,0 +1,50 @@
+package job
+
+import (
+	"91porn-server/common"
+	"91porn-server/common/log"
+	shortvideo "91porn-server/skd/job/sqlcollapser/shordvideo"
+	"91porn-server/skd/job/sqlcollapser/user"
+	"91porn-server/skd/job/sqlcollapser/vid"
+	"sync"
+)
+
+var syncSqlcollaspeIsRuining = false
+
+// start 定时任务 开启
+func runSyncSqlcollaspe() {
+	//防止重入
+	if syncSqlcollaspeIsRuining {
+		return
+	}
+	syncSqlcollaspeIsRuining = true
+	defer func() {
+		syncSqlcollaspeIsRuining = false
+	}()
+	doSyncSqlcollasper() //做折叠业务
+}
+
+// 做折叠业务
+func doSyncSqlcollasper() {
+	log.Info("SqlCollasper doSyncSqlcollasper start...")
+	defaultSyncDate, err := getDefaultSyncDate()
+	if err != nil {
+		log.Error("defaultStatDate invalid", log.E(err))
+		return
+	}
+	wg := sync.WaitGroup{}
+	wg.Add(3)
+	common.Go(func() {
+		defer wg.Done()
+		vid.RunSync(defaultSyncDate)
+	})
+	common.Go(func() {
+		defer wg.Done()
+		user.RunSync(defaultSyncDate)
+	})
+	common.Go(func() {
+		defer wg.Done()
+		shortvideo.RunSync(defaultSyncDate)
+	})
+	wg.Wait()
+}

skd/job/stat.go

@@ -0,0 +1,104 @@
+package job
+
+import (
+	"sync"
+	"time"
+
+	"91porn-server/common"
+	"91porn-server/common/log"
+	"91porn-server/skd/job/cache/tonecache"
+	"91porn-server/skd/job/stat/keyword"
+	"91porn-server/skd/job/stat/kwrank"
+	"91porn-server/skd/job/stat/rank"
+	"91porn-server/skd/job/stat/rechorder"
+	"91porn-server/skd/job/stat/statuser"
+	"91porn-server/skd/job/stat/tagplay"
+	"91porn-server/skd/job/stat/video"
+	"91porn-server/skd/job/stat/vidtotal"
+	"91porn-server/skd/job/stat/withorder"
+	"91porn-server/skd/skdg"
+)
+
+var (
+	statIsRunning     = false //true:统计任务运行中，反之停止，
+	statRankIsRunning = false
+)
+
+// 运行统计任务
+func runStat() {
+	//防止重入
+	if statIsRunning {
+		return
+	}
+	statIsRunning = true
+	defer func() {
+		statIsRunning = false
+	}()
+	doStat() //做统计业务
+}
+
+// 做统计业务
+func doStat() {
+	defaultStatDate, err := getDefaultStatDate()
+	if err != nil {
+		log.Error("defaultStatDate invalid", log.E(err))
+		return
+	}
+	methodM := []func(){
+		//主要用于app
+		func() { //"关键字"
+			keyword.RunStat(defaultStatDate)
+			kwrank.RunStat(defaultStatDate) //有执行顺序
+		},
+		func() { //"标签播放"
+			tagplay.RunStat(defaultStatDate)
+		},
+		func() { //"用户数据"
+			statuser.RunStat(defaultStatDate)
+		},
+		// func() { //"ranks"
+		//rankcache.Update()
+		// },
+		func() { //"tone"
+			tonecache.Update()
+		},
+		func() { //"视屏"
+			video.RunStat(defaultStatDate)
+			vidtotal.RunStat(defaultStatDate) //有执行顺序
+		},
+		func() { //"充值订单"
+			rechorder.RunStat(defaultStatDate)
+		},
+		func() { //"兑换订单"
+			withorder.RunStat(defaultStatDate)
+		},
+	}
+	time.Sleep(time.Duration(30) * time.Second)
+	wg := sync.WaitGroup{}
+	wg.Add(len(methodM))
+	for _, method := range methodM {
+		_method := method
+		common.Go(func() {
+			defer wg.Done()
+			_method()
+		})
+	}
+	wg.Wait()
+}
+
+func getDefaultStatDate() (time.Time, error) {
+	return time.ParseInLocation("2006-01-02", skdg.Conf.StartDate, time.Local)
+}
+
+// 运行排行榜
+func runRankStat() {
+	//防止重入
+	if statRankIsRunning {
+		return
+	}
+	statRankIsRunning = true
+	defer func() {
+		statRankIsRunning = false
+	}()
+	rank.RunStat()
+}

skd/job/stat/keyword/method.go

@@ -0,0 +1,59 @@
+package keyword
+
+import (
+	"91porn-server/common/constant"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/l/searchlogmod"
+	"91porn-server/models/s/kwstatmod"
+)
+
+type TimeRange = timerange.TimeRange
+
+func toKeywordIncDoc(realm constant.RealmType, data map[string]int64) []kwstatmod.KeywordIncDoc {
+	list := make([]kwstatmod.KeywordIncDoc, len(data))
+	i := 0
+	for k, v := range data {
+		word := k
+		count := v
+		list[i] = kwstatmod.KeywordIncDoc{
+			Realm: realm,
+			Word:  word,
+			Count: count,
+		}
+		i++
+	}
+	return list
+}
+
+func GetIncDocList(timeRange TimeRange) ([]kwstatmod.KeywordIncDoc, error) {
+	incDocList := []kwstatmod.KeywordIncDoc{}
+	//综合关键字搜索数
+	complexKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchComplex)
+	if err != nil {
+		return nil, err
+	}
+	complexIncDocList := toKeywordIncDoc(constant.SearchComplex, complexKeywordCountMap)
+	incDocList = append(incDocList, complexIncDocList...)
+	//视屏关键字搜索数
+	vidKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchSP)
+	if err != nil {
+		return nil, err
+	}
+	videoIncDocList := toKeywordIncDoc(constant.SearchSP, vidKeywordCountMap)
+	incDocList = append(incDocList, videoIncDocList...)
+	//标签关键字搜索数
+	tagKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchTag)
+	if err != nil {
+		return nil, err
+	}
+	tagIncDocList := toKeywordIncDoc(constant.SearchTag, tagKeywordCountMap)
+	incDocList = append(incDocList, tagIncDocList...)
+	//用户关键字搜索数
+	userKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchUser)
+	if err != nil {
+		return nil, err
+	}
+	userIncDocList := toKeywordIncDoc(constant.SearchUser, userKeywordCountMap)
+	incDocList = append(incDocList, userIncDocList...)
+	return incDocList, nil
+}

skd/job/stat/keyword/stat.go

@@ -0,0 +1,72 @@
+package keyword
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/db"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/kwstatmod"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/skd/skdg"
+)
+
+// RunStat
+func RunStat(defaultStatDate time.Time) {
+	log.Info("keyword stat running...")
+	defer log.Info("keyword stat finished...")
+	now := time.Now()
+	//从历史提交记录中获取上次记录时间
+	lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.KeywordStatJob, defaultStatDate) //第一次从默认记录时间开始统计
+	if err != nil {
+		log.Error("Keyword RunStat lastRecordTime faild!", log.E(err))
+		return
+	}
+	recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency == 0
+	//拆分时间区间并依次提交,均衡数据库读写压力
+	for recentMinute.After(lastStatAt) {
+		end := lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		//防止时间超出范围
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		subTimeRange := timerange.TimeRange{ //连续子切片
+			Head: lastStatAt,
+			Tail: end,
+		}
+		//此时的零日点
+		sumDate := timerange.LocDayRange(subTimeRange.Head).Head
+		//装配基础统计数据
+		incDocList, err := GetIncDocList(subTimeRange)
+		if err != nil {
+			log.Error("Keyword RunStat GetIncDocList faild!", log.E(err))
+			return
+		}
+		//提交数据
+		log.Info("keywordStat committing...", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+		if err = commit(sumDate, subTimeRange.Tail, incDocList); err != nil {
+			log.Error("Keyword RunStat commit faild!", log.E(err))
+			return
+		}
+		log.Info("keywordStat committed.", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+		lastStatAt = end
+	}
+}
+
+// commit 提交统计数据
+func commit(sumDate time.Time, recordTime time.Time, incDocList []kwstatmod.KeywordIncDoc) error {
+	opt := (&db.TransOpts{}).SetReEntry(10)
+	return skdg.StatDB.Trans(func(trans *db.MongoTool) (err error) {
+		if len(incDocList) != 0 {
+			if err = kwstatmod.ChangeStatTrans(trans, sumDate, recordTime, incDocList); err != nil {
+				return fmt.Errorf(" keyword commit ChangeStatTrans faild!, err:%+v\n", err)
+			}
+		}
+		//提交本次修改记录
+		if err = statrecordmod.UpsertOneTrans(trans, statrecordmod.KeywordStatJob, recordTime); err != nil {
+			return fmt.Errorf(" keyword commit UpsertOneTrans faild!, err:%+v\n", err)
+		}
+		return
+	}, opt)
+}

skd/job/stat/kwrank/method.go

@@ -0,0 +1,29 @@
+package kwrank
+
+import (
+	"time"
+
+	"91porn-server/models/s/kwrankmod"
+	"91porn-server/models/s/kwstatmod"
+)
+
+type RankSetDoc = kwrankmod.RankSetDoc
+
+func GetStatSetDocList(sumDate time.Time) ([]RankSetDoc, error) {
+	keywordCountMap, err := kwstatmod.KeywordCountMapBySumDate(sumDate)
+	if err != nil {
+		return nil, err
+	}
+	rankDocList := make([]RankSetDoc, len(keywordCountMap))
+	i := 0
+	for w, c := range keywordCountMap {
+		word := w
+		count := c
+		rankDocList[i] = RankSetDoc{
+			Word:  &word,
+			Count: &count,
+		}
+		i++
+	}
+	return rankDocList, nil
+}

skd/job/stat/kwrank/stat.go

@@ -0,0 +1,52 @@
+package kwrank
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/db"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/kwrankmod"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/skd/skdg"
+)
+
+// RunStat
+func RunStat(defaultStatDate time.Time) {
+	log.Info("keyword rank stat running...")
+	defer log.Info("keyword rank stat finished...")
+	now := time.Now()
+	sumDate := timerange.LocDayRange(now).Head //获取零日点
+	//装配基础统计数据
+	setList, err := GetStatSetDocList(sumDate)
+	if err != nil {
+		log.Error("kwrank RunStat fillStatSetDoc faild!", log.E(err))
+		return
+	}
+	//提交数据
+	log.Info("keyword rank stat committing...", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+	recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency  == 0
+	if err = commit(sumDate, recentMinute, setList); err != nil {
+		log.Error("kwrank RunStat commit faild!", log.E(err))
+		return
+	}
+	log.Info("keyword rank stat committed.", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+}
+
+// commit 提交统计数据
+func commit(sumDate time.Time, recordTime time.Time, setDocList []RankSetDoc) error {
+	opt := (&db.TransOpts{}).SetReEntry(10)
+	return skdg.StatDB.Trans(func(trans *db.MongoTool) (err error) {
+		if len(setDocList) != 0 {
+			if err = kwrankmod.ChangeStatTrans(trans, sumDate, recordTime, setDocList); err != nil {
+				return fmt.Errorf("kwrank commit ChangeStatTrans faild!, err:%+v\n", err)
+			}
+		}
+		//提交本次修改记录
+		if err = statrecordmod.UpsertOneTrans(trans, statrecordmod.KeywordRankStatJob, recordTime); err != nil {
+			return fmt.Errorf("kwrank commit UpsertOneTrans faild!, err:%+v\n", err)
+		}
+		return
+	}, opt)
+}

skd/job/stat/rank/stat.go

@@ -0,0 +1,53 @@
+package rank
+
+import (
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/hotstatmod"
+	"91porn-server/models/v/vidmod"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+// RunStat 统计实时排行榜
+func RunStat() {
+	log.Info("rank stat running...")
+	defer log.Info("rank stat finished...")
+	now := time.Now()
+	sumDate := timerange.LocDayRange(now).Head
+	infos, err := vidmod.GetTotalNewsList()
+	if err != nil {
+		return
+	}
+	mNow := calNowHot(infos)
+	mYesterday, err := hotstatmod.GetYesterdayHotValue(sumDate)
+	if err != nil {
+		return
+	}
+	mLastWeek, err := hotstatmod.GetLastWeekHotValue(sumDate)
+	if err != nil {
+		return
+	}
+	mLastMonth, _ := hotstatmod.GetLastMonthHotValue(sumDate)
+	mDayli := make(map[primitive.ObjectID]int)
+	mWeek := make(map[primitive.ObjectID]int)
+	mMonth := make(map[primitive.ObjectID]int)
+	for vid, v := range mNow {
+		mDayli[vid] = v - mYesterday[vid]
+		mWeek[vid] = v - mLastWeek[vid]
+		mMonth[vid] = v - mLastMonth[vid]
+	}
+	//入库
+	_ = hotstatmod.UpdateInsertHotValue(sumDate, mDayli, mWeek, mMonth, mNow)
+}
+
+func calNowHot(infos []*vidmod.VideoModel) map[primitive.ObjectID]int {
+	mNow := make(map[primitive.ObjectID]int)
+	for _, v := range infos {
+		value := 1*v.FakePlayCount + 4*v.FakeLikeCount + 2*v.FakeCommentCount + v.FakeShareCount
+		mNow[v.ID] = value
+	}
+	return mNow
+}

skd/job/stat/rechorder/method.go

@@ -0,0 +1,138 @@
+package rechorder
+
+import (
+	"time"
+
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statordermod"
+	"91porn-server/models/v/rchgordmod"
+)
+
+type RechargeOrderStateDoc = statordermod.RechargeOrderStateDoc
+type RecOrderStateMap = statordermod.RecOrderStateMap
+
+func fillRecOrderStatDocMap(sumDate time.Time, timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
+	//订单数 OrderCount
+	if err := fillRecOrderCount(timeRange, docMap); err != nil {
+		return err
+	}
+	//完成订单数 CompleteOrderCount
+	if err := fillRecCompleteOrderCount(timeRange, docMap); err != nil {
+		return err
+	}
+	//订单金额 OrderAmount
+	if err := fillRecOrderAmount(timeRange, docMap); err != nil {
+		return err
+	}
+	if err := fillRecCompleteOrderPayMoney(timeRange, docMap); err != nil {
+		return err
+	}
+	//用户已支付订单数
+	if err := fillRecPaidOrderCount(timeRange, docMap); err != nil {
+		return err
+	}
+	//用户已支付金额
+	return fillRecPaidMoney(timeRange, docMap)
+}
+
+// fillRecOrderCount fillRecOrderCount
+func fillRecOrderCount(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
+	orderCountMap, err := rchgordmod.ChannelOrderCountMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range orderCountMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &RechargeOrderStateDoc{}
+		}
+		val := v
+		docM[channel].OrderCount = &val
+	}
+	return nil
+}
+
+// fillRecCompleteOrderCount fillRecCompleteOrderCount}
+func fillRecCompleteOrderCount(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
+	completeOrderCountMap, err := rchgordmod.ChannelCompleteOrderCountMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range completeOrderCountMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &RechargeOrderStateDoc{}
+		}
+		val := v
+		docM[channel].CompleteOrderCount = &val
+	}
+	return nil
+}
+
+// fillRecOrderAmount fillRecOrderAmount
+func fillRecOrderAmount(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
+	orderAmountMap, err := rchgordmod.ChannelOrderMoneyMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range orderAmountMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &RechargeOrderStateDoc{}
+		}
+		val := v
+		docM[channel].OrderAmount = &val
+	}
+	return nil
+}
+
+// fillRecCompleteOrderPayMoney fillRecCompleteOrderPayMoney
+func fillRecCompleteOrderPayMoney(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
+	payMoneyMap, err := rchgordmod.ChannelCompleteOrderPayMoneyMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range payMoneyMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &RechargeOrderStateDoc{}
+		}
+		val := v
+		docM[channel].CompleteOrderAmount = &val
+	}
+	return nil
+}
+
+// fillRecPaidOrderCount fillRecPaidOrderCount
+func fillRecPaidOrderCount(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
+	paiedOrderCountMap, err := rchgordmod.ChannelPaymentOrderCountMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range paiedOrderCountMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &RechargeOrderStateDoc{}
+		}
+		val := v
+		docM[channel].PaidOrderCount = &val
+	}
+	return nil
+}
+
+// fillRecPaidMoney fillRecPaidMoney
+func fillRecPaidMoney(timeRange timerange.TimeRange, docMap *RecOrderStateMap) error {
+	paiedMoneyMap, err := rchgordmod.ChannelPaymentMoneyMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range paiedMoneyMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &RechargeOrderStateDoc{}
+		}
+		val := v
+		docM[channel].PaidAmount = &val
+	}
+	return nil
+}

skd/job/stat/rechorder/stat.go

@@ -0,0 +1,97 @@
+package rechorder
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/db"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statordermod"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/skd/skdg"
+)
+
+func RunStat(defaultStatDate time.Time) {
+	log.Info("rechorderStat stat running...")
+	defer log.Info("rechorderStat stat finished...")
+	now := time.Now()
+	//从历史提交记录中获取上次记录时间
+	lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.RechargeOrderStatJob, defaultStatDate) //第一次从默认记录时间开始统计
+	if err != nil {
+		log.Error("rechorder RunStat lastRecordTime faild!", log.E(err))
+		return
+	}
+	//过去几天的数据按一日为单位统计，帮助数据恢复
+	dayHead := timerange.LocDayRange(now).Head
+	for dayHead.After(lastStatAt) {
+		end := lastStatAt.AddDate(0, 0, 1)
+		//防止时间超出范围
+		if end.After(dayHead) {
+			end = dayHead
+		}
+		subTimeRange := timerange.TimeRange{ //连续子切片
+			Head: lastStatAt,
+			Tail: end,
+		}
+		if err := statByTimeRange(subTimeRange); err != nil {
+			log.Error("rechorder RunStat statByTimeRange faild!", log.E(err))
+			return
+		}
+		lastStatAt = end
+	}
+	//当天的数据按照5分钟统计
+	recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency  == 0
+	//拆分时间区间并依次提交,控制数据库读写压力
+	for recentMinute.After(lastStatAt) {
+		end := lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		//防止时间超出范围
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		subTimeRange := timerange.TimeRange{ //连续子切片
+			Head: lastStatAt,
+			Tail: end,
+		}
+		if err := statByTimeRange(subTimeRange); err != nil {
+			log.Error("rechorder RunStat statByTimeRange faild!", log.E(err))
+			return
+		}
+		lastStatAt = end
+	}
+}
+
+func statByTimeRange(timeRange timerange.TimeRange) error {
+	sumDate := timerange.LocDayRange(timeRange.Head).Head //获取此时的零日点
+	//装配渠道统计数据
+	recOrderStateMap := make(RecOrderStateMap)
+	if err := fillRecOrderStatDocMap(sumDate, timeRange, &recOrderStateMap); err != nil {
+		log.Error("rechorder RunStat fillRecOrderStatDocMap faild!", log.E(err))
+		return err
+	}
+	//提交渠道统计数据
+	log.Info("rechorderStat recharge committing...", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+	if err := commitRec(sumDate, timeRange.Tail, recOrderStateMap); err != nil {
+		log.Error("rechorderStat RunStat commitRec faild!", log.E(err))
+		return err
+	}
+	log.Info("rechorderStat recharge committed.", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+	return nil
+}
+
+// commit 提交订单统计数据
+func commitRec(sumDate time.Time, recordTime time.Time, docMap RecOrderStateMap) error {
+	opt := (&db.TransOpts{}).SetReEntry(10)
+	return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
+		if len(docMap) != 0 {
+			if err := statordermod.ChangeRechargeStatTrans(trans, sumDate, recordTime, docMap); err != nil {
+				return fmt.Errorf("stat commitRec ChangeRechargeStatTrans faild!, err:%+v\n", err)
+			}
+		}
+		//提交本次修改记录
+		if err := statrecordmod.UpsertOneTrans(trans, statrecordmod.RechargeOrderStatJob, recordTime); err != nil {
+			return fmt.Errorf("stat commitRec UpsertOneTrans faild!, err:%+v\n", err)
+		}
+		return nil
+	}, opt)
+}

skd/job/stat/statuser/method.go

@@ -0,0 +1,91 @@
+package statuser
+
+import (
+	"time"
+
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/l/payvidlgmod"
+	"91porn-server/models/s/statusermod"
+	"91porn-server/models/v/vidmod"
+)
+
+type TimeRange = timerange.TimeRange
+
+type IncDoc = statusermod.IncDoc
+
+type SetDoc = statusermod.SetDoc
+
+type Itemtype = statusermod.Itemtype
+
+const (
+	SetType string = "set"
+	IncType string = "inc"
+)
+
+type Method struct {
+	MethodType          string
+	GetIncDocListMethod func(timeRange TimeRange) ([]IncDoc, error)
+	GetSetDocListMethod func(timeRange TimeRange) ([]SetDoc, error)
+	IncCommitMethod     func(incDocList []IncDoc, name string, recordAt time.Time) error
+	SetCommitMethod     func(setDocList []SetDoc, name string, recordAt time.Time) error
+}
+
+var methodMap = map[Itemtype]Method{
+	//视频收益
+	statusermod.VidIncome: {MethodType: IncType, GetIncDocListMethod: getIncomneDocList, IncCommitMethod: commitIncDoc},
+	//上传数量
+	statusermod.UploadCount: {MethodType: SetType, GetSetDocListMethod: getUploadCountDocList, SetCommitMethod: commitSetDoc},
+}
+
+func getIncomneDocList(timeRange TimeRange) ([]IncDoc, error) {
+	m, err := payvidlgmod.PublisherIncomeMapByTime(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return nil, err
+	}
+	list := make([]IncDoc, len(m))
+	i := 0
+	for uid, income := range m {
+		list[i] = IncDoc{
+			UID:      uid,
+			Item:     statusermod.VidIncome,
+			Count:    income,
+			RecordAt: timeRange.Tail,
+		}
+		i++
+	}
+	return list, nil
+}
+
+func getUploadCountDocList(timeRange TimeRange) ([]SetDoc, error) {
+	mats := []vidmod.Matcher{
+		(&vidmod.StatusMatch{Status: vidmod.CheckPass}).New(),
+		(&vidmod.ReviewAtGTEAndLTMatch{GTE: &timeRange.Head, LT: &timeRange.Tail}).New(),
+	}
+	publisherIDs, err := vidmod.PublisherIDList(mats...)
+	if err != nil {
+		return nil, err
+	}
+	if len(publisherIDs) == 0 {
+		return nil, nil
+	}
+	mats = []vidmod.Matcher{
+		(&vidmod.StatusMatch{Status: vidmod.CheckPass}).New(),
+		(&vidmod.PublisherIDInMatch{PublisherIDList: publisherIDs}).New(),
+	}
+	m, err := vidmod.CountMapByPublisherID(mats...)
+	if err != nil {
+		return nil, err
+	}
+	list := make([]SetDoc, len(m))
+	i := 0
+	for uid, uploadCount := range m {
+		list[i] = SetDoc{
+			UID:      uid,
+			Item:     statusermod.UploadCount,
+			Count:    uploadCount,
+			RecordAt: timeRange.Tail,
+		}
+		i++
+	}
+	return list, nil
+}

skd/job/stat/statuser/stat.go

@@ -0,0 +1,143 @@
+package statuser
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/db"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/s/statusermod"
+	"91porn-server/skd/skdg"
+)
+
+func RunStat(defaultStatDate time.Time) {
+	log.Info("user stat running...")
+	defer log.Info("user stat finished...")
+	defaultTimeMap := map[string]time.Time{
+		string(statusermod.VidIncome):   defaultStatDate,
+		string(statusermod.UploadCount): defaultStatDate,
+	}
+	//从历史提交记录中获取上次记录时间
+	lastStatAtMap, err := statrecordmod.LastRecordTimeMap(statrecordmod.UserStatJob, defaultTimeMap) //第一次从默认记录时间开始统计
+	if err != nil {
+		log.Error("user stat statuser LastRecordTimeMap faild!", log.E(err))
+		return
+	}
+	now := time.Now()
+	dayHead := timerange.LocDayRange(now).Head
+	for item, lastStatAt := range lastStatAtMap {
+		_lastStatAt := lastStatAt
+		//过去几天的数据按一日为单位统计，帮助数据恢复
+		for dayHead.After(_lastStatAt) {
+			end := _lastStatAt.AddDate(0, 0, 1)
+			//防止时间超出范围
+			if end.After(dayHead) {
+				end = dayHead
+			}
+			subTimeRange := timerange.TimeRange{ //连续子切片
+				Head: _lastStatAt,
+				Tail: end,
+			}
+			if err = statByTimeRange(item, subTimeRange); err != nil {
+				log.Error("user stat statuser statByTimeRange 1 faild!", log.E(err))
+				return
+			}
+			_lastStatAt = end
+		}
+		lastStatAtMap[item] = _lastStatAt
+	}
+	recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency  == 0
+	//拆分时间区间并依次提交,控制数据库读写压力
+	for item, lastStatAt := range lastStatAtMap {
+		_lastStatAt := lastStatAt
+		for recentMinute.After(_lastStatAt) {
+			end := _lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+			//防止时间超出范围
+			if end.After(recentMinute) {
+				end = recentMinute
+			}
+			subTimeRange := timerange.TimeRange{ //连续子切片
+				Head: _lastStatAt,
+				Tail: end,
+			}
+			if err = statByTimeRange(item, subTimeRange); err != nil {
+				log.Error("user stat statuser statByTimeRange 2 faild!", log.E(err))
+				return
+			}
+			_lastStatAt = end
+		}
+		lastStatAtMap[item] = _lastStatAt
+	}
+}
+
+func statByTimeRange(item string, timeRange timerange.TimeRange) error {
+	method, ok := methodMap[Itemtype(item)]
+	if !ok {
+		return nil
+	}
+	if method.MethodType == IncType {
+		//装配INC基础统计数据
+		incDocList, err := method.GetIncDocListMethod(timeRange)
+		if err != nil {
+			log.Error("user stat statByTimeRange GetIncDocListMethod faild!", log.Any("item", item), log.E(err))
+			return err
+		}
+		//提交Inc统计数据
+		log.Info("user stat statByTimeRange IncCommitMethod committing...", log.Any("item", item))
+		if err = method.IncCommitMethod(incDocList, item, timeRange.Tail); err != nil {
+			log.Error("user stat statByTimeRange IncCommitMethod faild!", log.Any("item", item), log.E(err))
+			return err
+		}
+		log.Info("user stat statByTimeRange IncCommitMethod committed.", log.Any("item", item))
+	} else if method.MethodType == SetType {
+		//装配SET基础统计数据
+		setDocList, err := method.GetSetDocListMethod(timeRange)
+		if err != nil {
+			log.Error("user stat statByTimeRange GetSetDocListMethod faild!", log.Any("item", item), log.E(err))
+			return err
+		}
+		//提交SET统计数据
+		log.Info("user stat statByTimeRange SetCommitMethod committing...", log.Any("item", item))
+		if err = method.SetCommitMethod(setDocList, item, timeRange.Tail); err != nil {
+			log.Error("user stat statByTimeRange SetCommitMethod faild!", log.Any("item", item), log.E(err))
+			return err
+		}
+		log.Info("user stat statByTimeRange SetCommitMethod committed.", log.Any("item", item))
+	}
+	return nil
+}
+
+func commitIncDoc(incDocList []IncDoc, item string, recordAt time.Time) error {
+	opt := (&db.TransOpts{}).SetReEntry(10)
+	return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
+		if len(incDocList) != 0 {
+			if err := statusermod.ChangeIncStatTrans(trans, incDocList); err != nil {
+				return fmt.Errorf("user stat commitCount ChangeIncStatTrans faild!, err:%+v\n", err)
+			}
+		}
+		//提交本次修改记录
+		if err := statrecordmod.UpsertOneItemTrans(trans, statrecordmod.UserStatJob, item, recordAt); err != nil {
+			return fmt.Errorf("user stat commitIncDoc UpsertOneItemTrans faild!, err:%+v\n", err)
+		}
+		return nil
+	}, opt)
+}
+
+func commitSetDoc(setDocList []SetDoc, item string, recordAt time.Time) error {
+	opt := (&db.TransOpts{}).SetReEntry(10)
+	return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
+		if len(setDocList) != 0 {
+			err := statusermod.ChangeSetStatTrans(trans, setDocList)
+			if err != nil {
+				return fmt.Errorf("user stat commitCount ChangeSetStatTrans faild!, err:%+v\n", err)
+			}
+		}
+		//提交本次修改记录
+		if err := statrecordmod.UpsertOneItemTrans(trans, statrecordmod.UserStatJob, item, recordAt); err != nil {
+			return fmt.Errorf("user stat commitSetDoc UpsertOneItemTrans faild!, item: %s+v, err:%+v\n", item, err)
+		}
+		return nil
+	}, opt)
+}

skd/job/stat/tagplay/method.go

@@ -0,0 +1,25 @@
+package tagplay
+
+import (
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/l/playlgmod"
+	"91porn-server/models/s/stattagplaymod"
+)
+
+type IncDoc = stattagplaymod.IncDoc
+
+func GetTagPlayIncDocList(timeRange timerange.TimeRange) ([]IncDoc, error) {
+	playLogList, err := playlgmod.GetTagPlayByTimeRange(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return []IncDoc{}, err
+	}
+	tagPlayList := make([]IncDoc, len(playLogList))
+	for i, v := range playLogList {
+		tagPlayList[i] = IncDoc{
+			VideoID:   v.GroupID.VideoID,
+			TagID:     v.GroupID.TagID,
+			PlayCount: v.PlayCount,
+		}
+	}
+	return tagPlayList, nil
+}

skd/job/stat/tagplay/stat.go

@@ -0,0 +1,71 @@
+package tagplay
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/db"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/s/stattagplaymod"
+	"91porn-server/skd/skdg"
+)
+
+func RunStat(defaultStatDate time.Time) {
+	log.Info("tag play stat start...")
+	defer log.Info("tag play stat end...")
+	// 获取上次统计的记录时间
+	lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.TagPlayStatJob, defaultStatDate) //第一次从默认记录时间开始统计
+	if err != nil {
+		log.Error("tag play RunStat lastRecordTime faild!", log.E(err))
+		return
+	}
+	now := time.Now()
+	// 按照每个钟的5分钟刻度统计: 0,05,10,15,20,25,30,35,40,45,50,55
+	// 获取最近的时间刻度
+	recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastStatAt) {
+		end := lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		//防止时间超出范围
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		// 时间区间5分钟
+		subTimeRange := timerange.TimeRange{
+			Head: lastStatAt,
+			Tail: end,
+		}
+		//5分钟内的统计数据
+		incDocList, err := GetTagPlayIncDocList(subTimeRange)
+		if err != nil {
+			log.Error("tag play RunStat GetTagPlayList err!", log.E(err))
+			return
+		}
+		//每次循环都应提交
+		log.Info("tag play stat committing...")
+		if err = commit(incDocList, subTimeRange.Tail); err != nil {
+			log.Error("tag play RunStat commit err! ", log.E(err))
+			return
+		}
+		log.Info("tag play stat committed.")
+		// 统计数据
+		lastStatAt = end
+	}
+}
+
+func commit(statData []IncDoc, recordTime time.Time) error {
+	opt := (&db.TransOpts{}).SetReEntry(10)
+	return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
+		if len(statData) != 0 {
+			if err := stattagplaymod.ChangeStatTrans(trans, statData); err != nil {
+				return fmt.Errorf("tag play commit ChangeStatTrans faild!, err:%+v\n", err)
+			}
+		}
+		//提交本次修改记录
+		if err := statrecordmod.UpsertOneTrans(trans, statrecordmod.TagPlayStatJob, recordTime); err != nil {
+			return fmt.Errorf("tag play commit UpsertOneTrans faild!, err:%+v\n", err)
+		}
+		return nil
+	}, opt)
+}

skd/job/stat/video/method.go

@@ -0,0 +1,130 @@
+package video
+
+import (
+	"time"
+
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/l/payvidlgmod"
+	"91porn-server/models/l/playlgmod"
+	"91porn-server/models/s/statvidmod"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ObjectID = primitive.ObjectID
+
+type TimeRange = timerange.TimeRange
+
+type VideoStatIncDoc = statvidmod.VideoStatIncDoc
+
+type VideoStatSetDoc = statvidmod.VideoStatSetDoc
+
+type VideoStatDoc = statvidmod.VideoStatDoc
+
+type VideoStatDocMap = statvidmod.VideoStatDocMap //ObjectID->VideoStatDoc
+
+// fillPayCoin
+func fillPayCoin(timeRange TimeRange, docMap *VideoStatDocMap) error {
+	incomeMap, err := payvidlgmod.PayCoinMapByTime(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for vid, v := range incomeMap {
+		if _, existed := docM[vid]; !existed {
+			docM[vid] = &VideoStatDoc{}
+		}
+		val := v
+		docM[vid].Income = &val
+	}
+	return nil
+}
+
+// fillPayCount
+func fillPayCount(timeRange TimeRange, docMap *VideoStatDocMap) error {
+	payCountMap, err := payvidlgmod.PayCountMapByTime(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for vid, v := range payCountMap {
+		if _, existed := docM[vid]; !existed {
+			docM[vid] = &VideoStatDoc{}
+		}
+		val := v
+		docM[vid].PayCount = &val
+	}
+	return nil
+}
+
+// fillPlayCount
+func fillPlayCount(timeRange TimeRange, docMap *VideoStatDocMap) error {
+	playCountMap, err := playlgmod.PlayCountMapByTime(timeRange.Head, timeRange.Tail, []int{0, 1, 2}) //0，免费 1付费 2.试看
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for vid, v := range playCountMap {
+		if _, existed := docM[vid]; !existed {
+			docM[vid] = &VideoStatDoc{}
+		}
+		val := v
+		docM[vid].PlayCount = &val
+	}
+	return nil
+}
+
+// fillPayPlayCount
+func fillPayPlayCount(timeRange TimeRange, docMap *VideoStatDocMap) error {
+	payPlayCountMap, err := playlgmod.PlayCountMapByTime(timeRange.Head, timeRange.Tail, []int{1}) //0，免费 1付费 2.试看
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for vid, v := range payPlayCountMap {
+		if _, existed := docM[vid]; !existed {
+			docM[vid] = &VideoStatDoc{}
+		}
+		val := v
+		docM[vid].PayPlayCount = &val
+	}
+	return nil
+}
+
+func fillTax(timeRange TimeRange, docMap *VideoStatDocMap) error {
+	payPlayCountMap, err := payvidlgmod.TaxAmountMapByTime(timeRange.Head, timeRange.Tail) //0，免费 1付费 2.试看
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for vid, v := range payPlayCountMap {
+		if _, existed := docM[vid]; !existed {
+			docM[vid] = &VideoStatDoc{}
+		}
+		val := v
+		docM[vid].Tax = &val
+	}
+	return nil
+}
+
+// fillStatDocMap
+func fillStatDocMap(sumDate time.Time, timeRange TimeRange, docMap *VideoStatDocMap) error {
+	//收入
+	if err := fillPayCoin(timeRange, docMap); err != nil {
+		return err
+	}
+	//购买次数
+	if err := fillPayCount(timeRange, docMap); err != nil {
+		return err
+	}
+	//视屏播放次数
+	if err := fillPlayCount(timeRange, docMap); err != nil {
+		return err
+	}
+	//付费视屏播放次数
+	if err := fillPayPlayCount(timeRange, docMap); err != nil {
+		return err
+	}
+	//税
+	return fillTax(timeRange, docMap)
+}

skd/job/stat/video/stat.go

@@ -0,0 +1,97 @@
+package video
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/db"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/s/statvidmod"
+	"91porn-server/skd/skdg"
+)
+
+func RunStat(defaultStatDate time.Time) {
+	log.Info("video stat running...")
+	defer log.Info("video stat finished...")
+	now := time.Now()
+	//从历史提交记录中获取上次记录时间
+	lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.VideoStatJob, defaultStatDate) //第一次从默认记录时间开始统计
+	if err != nil {
+		log.Error("video stat RunVideoStat lastRecordTime faild!", log.E(err))
+		return
+	}
+	//过去几天的数据按一日为单位统计，帮助数据恢复
+	dayHead := timerange.LocDayRange(now).Head
+	for dayHead.After(lastStatAt) {
+		end := lastStatAt.AddDate(0, 0, 1)
+		//防止时间超出范围
+		if end.After(dayHead) {
+			end = dayHead
+		}
+		subTimeRange := timerange.TimeRange{ //连续子切片
+			Head: lastStatAt,
+			Tail: end,
+		}
+		if err = statByTimeRange(subTimeRange); err != nil {
+			log.Error("video RunStat statByTimeRange faild!", log.E(err))
+			return
+		}
+		lastStatAt = end
+	}
+	//当天的数据按照5分钟统计
+	recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency  == 0
+	//拆分时间区间并依次提交,控制数据库读写压力
+	for recentMinute.After(lastStatAt) {
+		end := lastStatAt.Add(statrecordmod.OneMinuteScale * time.Minute)
+		//防止时间超出范围
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		subTimeRange := timerange.TimeRange{ //连续子切片
+			Head: lastStatAt,
+			Tail: end,
+		}
+		if err = statByTimeRange(subTimeRange); err != nil {
+			log.Error("video RunStat statByTimeRange faild!", log.E(err))
+			return
+		}
+		lastStatAt = end
+	}
+}
+
+func statByTimeRange(timeRange timerange.TimeRange) error {
+	sumDate := timerange.LocDayRange(timeRange.Head).Head //获取此时的零日点
+	//装配基础统计数据
+	statDocMap := make(VideoStatDocMap)
+	if err := fillStatDocMap(sumDate, timeRange, &statDocMap); err != nil {
+		log.Error("video stat RunVideoStat fillStatDocMap faild!", log.E(err))
+		return err
+	}
+	//提交渠道统计数据
+	log.Info("videoStat committing...", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+	if err := commit(sumDate, timeRange.Tail, statDocMap); err != nil {
+		log.Error("video stat RunVideoStat commit faild!", log.E(err))
+		return err
+	}
+	log.Info("videoStat committed.", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+	return nil
+}
+
+// commit 提交统计数据
+func commit(sumDate time.Time, recordTime time.Time, docMap VideoStatDocMap) error {
+	opt := (&db.TransOpts{}).SetReEntry(10)
+	return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
+		if len(docMap) != 0 {
+			if err := statvidmod.ChangeStatTrans(trans, sumDate, recordTime, docMap); err != nil {
+				return fmt.Errorf("video stat commit ChangeStatTrans faild!, err:%+v\n", err)
+			}
+		}
+		//提交本次修改记录
+		if err := statrecordmod.UpsertOneTrans(trans, statrecordmod.VideoStatJob, recordTime); err != nil {
+			return fmt.Errorf("video stat commit UpsertOneTrans faild!, err:%+v\n", err)
+		}
+		return nil
+	}, opt)
+}

skd/job/stat/vidtotal/method.go

@@ -0,0 +1,46 @@
+package vidtotal
+
+import (
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statvidmod"
+	"91porn-server/models/s/statvidtotalmod"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+type ObjectID = primitive.ObjectID
+
+type TimeRange = timerange.TimeRange
+
+type VideoTotalIncDoc = statvidtotalmod.IncDoc
+
+type VideoTotalIncDocMap = statvidtotalmod.IncDocMap
+
+type RecordAtGTEAndLTMatch = statvidmod.RecordAtGTEAndLTMatch
+
+func getVideoTotalIncDocMap(timeRange TimeRange) (VideoTotalIncDocMap, error) {
+	recordAtGTEAndLTMatch := RecordAtGTEAndLTMatch{GTE: &timeRange.Head, LT: &timeRange.Tail}
+	groupList, err := statvidmod.SumByRecordAt(recordAtGTEAndLTMatch)
+	if err != nil {
+		return nil, err
+	}
+	m := toVideoTotalIncDocMap(groupList)
+	return m, nil
+}
+
+func toVideoTotalIncDocMap(groupList []statvidmod.SumGroup) VideoTotalIncDocMap {
+	m := make(VideoTotalIncDocMap, len(groupList))
+	for _, v := range groupList {
+		_, ok := m[v.Vid]
+		if !ok {
+			m[v.Vid] = &VideoTotalIncDoc{}
+		}
+		doc := m[v.Vid]
+		doc.Income = v.Income
+		doc.PayCount = v.PayCount
+		doc.PlayCount = v.PlayCount
+		doc.PayPlayCount = v.PayPlayCount
+		doc.Tax = v.Tax
+	}
+	return m
+}

skd/job/stat/vidtotal/stat.go

@@ -0,0 +1,96 @@
+package vidtotal
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/db"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/s/statvidtotalmod"
+	"91porn-server/skd/skdg"
+)
+
+func RunStat(defaultStatDate time.Time) {
+	log.Info("video total stat running...")
+	defer log.Info("video total stat finished...")
+	now := time.Now()
+	//从历史提交记录中获取上次记录时间
+	lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.VideoTotalStatJob, defaultStatDate) //第一次从默认记录时间开始统计
+	if err != nil {
+		log.Error("video total stat RunVideoStat lastRecordTime faild!", log.E(err))
+		return
+	}
+	//过去几天的数据按一日为单位统计，帮助数据恢复
+	dayHead := timerange.LocDayRange(now).Head
+	for dayHead.After(lastStatAt) {
+		end := lastStatAt.AddDate(0, 0, 1)
+		//防止时间超出范围
+		if end.After(dayHead) {
+			end = dayHead
+		}
+		subTimeRange := timerange.TimeRange{ //连续子切片
+			Head: lastStatAt,
+			Tail: end,
+		}
+		if err = statByTimeRange(subTimeRange); err != nil {
+			log.Error("video total RunStat statByTimeRange faild!", log.E(err))
+			return
+		}
+		lastStatAt = end
+	}
+	//当天的数据按照5分钟统计
+	recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency  == 0
+	//拆分时间区间并依次提交,控制数据库读写压力
+	for recentMinute.After(lastStatAt) {
+		end := lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		//防止时间超出范围
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		subTimeRange := timerange.TimeRange{ //连续子切片
+			Head: lastStatAt,
+			Tail: end,
+		}
+		if err = statByTimeRange(subTimeRange); err != nil {
+			log.Error("video total RunStat statByTimeRange faild!", log.E(err))
+			return
+		}
+		lastStatAt = end
+	}
+}
+
+func statByTimeRange(timeRange timerange.TimeRange) error {
+	//装配基础统计数据
+	statDocMap, err := getVideoTotalIncDocMap(timeRange)
+	if err != nil {
+		log.Error("video total stat RunVideoStat fillStatDocMap faild!", log.E(err))
+		return err
+	}
+	//提交视屏总计数据
+	log.Info("videoTotalStat committing...")
+	if err = commit(timeRange.Tail, statDocMap); err != nil {
+		log.Error("video total stat RunVideoStat commit faild!", log.E(err))
+		return err
+	}
+	log.Info("videoTotalStat committed.")
+	return nil
+}
+
+// commit 提交统计数据
+func commit(recordTime time.Time, docMap VideoTotalIncDocMap) error {
+	opt := (&db.TransOpts{}).SetReEntry(10)
+	return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
+		if len(docMap) != 0 {
+			if err := statvidtotalmod.ChangeStatTrans(trans, recordTime, docMap); err != nil {
+				return fmt.Errorf("video total stat commit ChangeStatTrans faild!, err:%+v\n", err)
+			}
+		}
+		//提交本次修改记录
+		if err := statrecordmod.UpsertOneTrans(trans, statrecordmod.VideoTotalStatJob, recordTime); err != nil {
+			return fmt.Errorf("video total stat commit UpsertOneTrans faild!, err:%+v\n", err)
+		}
+		return nil
+	}, opt)
+}

skd/job/stat/withorder/method.go

@@ -0,0 +1,139 @@
+package withorder
+
+import (
+	"time"
+
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statordermod"
+	"91porn-server/models/v/wdordmod"
+)
+
+type WithDrawOrderStateDoc = statordermod.WithDrawOrderStateDoc
+type WitOrderStateMap = statordermod.WitOrderStateMap
+
+func fillWitOrderStatDocMap(sumDate time.Time, timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
+	//订单数 OrderCount
+	if err := fillWitOrderCount(timeRange, docMap); err != nil {
+		return err
+	}
+	//完成订单数 CompleteOrderCount
+	if err := fillWitCompleteOrderCount(timeRange, docMap); err != nil {
+		return err
+	}
+	//订单金额 OrderAmount
+	if err := fillWitOrderAmount(timeRange, docMap); err != nil {
+		return err
+	}
+	//实际兑换金额 CompleteOrderAmount
+	if err := fillWitCompleteOrderPayAmount(timeRange, docMap); err != nil {
+		return err
+	}
+	//三方已出款订单数 fillWitPaiedOrderCount
+	if err := fillWitPaiedOrderCount(timeRange, docMap); err != nil {
+		return err
+	}
+	//三方已出款金额 fillWitPaiedMoney
+	return fillWitPaiedMoney(timeRange, docMap)
+}
+
+// fillWitOrderCount fillWitOrderCount
+func fillWitOrderCount(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
+	orderCountMap, err := wdordmod.ChannelOrderCountMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range orderCountMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &WithDrawOrderStateDoc{}
+		}
+		val := v
+		docM[channel].OrderCount = &val
+	}
+	return nil
+}
+
+// fillWitCompleteOrderCount fillWitCompleteOrderCount
+func fillWitCompleteOrderCount(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
+	completeOrderCountMap, err := wdordmod.ChannelCompleteOrderCountMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range completeOrderCountMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &WithDrawOrderStateDoc{}
+		}
+		val := v
+		docM[channel].CompleteOrderCount = &val
+	}
+	return nil
+}
+
+// fillWitOrderAmount fillWitOrderAmount
+func fillWitOrderAmount(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
+	orderAmountMap, err := wdordmod.ChannelOrderMoneyMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range orderAmountMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &WithDrawOrderStateDoc{}
+		}
+		val := v
+		docM[channel].OrderAmount = &val
+	}
+	return nil
+}
+
+// fillWitCompleteOrderPayAmount fillWitCompleteOrderPayAmount
+func fillWitCompleteOrderPayAmount(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
+	completeOrderAmountMap, err := wdordmod.ChannelCompleteOrderPayMoneyMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range completeOrderAmountMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &WithDrawOrderStateDoc{}
+		}
+		val := v
+		docM[channel].CompleteOrderAmount = &val
+	}
+	return nil
+}
+
+// fillWitPaiedOrderCount fillWitPaiedOrderCount
+func fillWitPaiedOrderCount(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
+	receivedOrderCountMap, err := wdordmod.ChannelReceivedOrderCountMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range receivedOrderCountMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &WithDrawOrderStateDoc{}
+		}
+		val := v
+		docM[channel].PaidOrderCount = &val
+	}
+	return nil
+}
+
+// fillWitPaiedMoney fillWitPaiedMoney
+func fillWitPaiedMoney(timeRange timerange.TimeRange, docMap *WitOrderStateMap) error {
+	receivedMoneyMap, err := wdordmod.ChannelReceivedMoneyMap(timeRange.Head, timeRange.Tail)
+	if err != nil {
+		return err
+	}
+	docM := *docMap
+	for channel, v := range receivedMoneyMap {
+		if _, existed := docM[channel]; !existed {
+			docM[channel] = &WithDrawOrderStateDoc{}
+		}
+		val := v
+		docM[channel].PaidAmount = &val
+	}
+	return nil
+}

skd/job/stat/withorder/stat.go

@@ -0,0 +1,97 @@
+package withorder
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/db"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statordermod"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/skd/skdg"
+)
+
+func RunStat(defaultStatDate time.Time) {
+	log.Info("withorder stat running...")
+	defer log.Info("withorder stat finished...")
+	now := time.Now()
+	//从历史提交记录中获取上次记录时间
+	lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.WithdrawOrderStatJob, defaultStatDate) //第一次从默认记录时间开始统计
+	if err != nil {
+		log.Error("withorder RunStat lastRecordTime faild!", log.E(err))
+		return
+	}
+	//过去几天的数据按一日为单位统计，帮助数据恢复
+	dayHead := timerange.LocDayRange(now).Head
+	for dayHead.After(lastStatAt) {
+		end := lastStatAt.AddDate(0, 0, 1)
+		//防止时间超出范围
+		if end.After(dayHead) {
+			end = dayHead
+		}
+		subTimeRange := timerange.TimeRange{ //连续子切片
+			Head: lastStatAt,
+			Tail: end,
+		}
+		if err = statByTimeRange(subTimeRange); err != nil {
+			log.Error("withorder RunStat statByTimeRange 1 faild!", log.E(err))
+			return
+		}
+		lastStatAt = end
+	}
+	//当天的数据按照5分钟统计
+	recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency  == 0
+	//拆分时间区间并依次提交,控制数据库读写压力
+	for recentMinute.After(lastStatAt) {
+		end := lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		//防止时间超出范围
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		subTimeRange := timerange.TimeRange{ //连续子切片
+			Head: lastStatAt,
+			Tail: end,
+		}
+		if err = statByTimeRange(subTimeRange); err != nil {
+			log.Error("withorder RunStat statByTimeRange 2 faild!", log.E(err))
+			return
+		}
+		lastStatAt = end
+	}
+}
+
+func statByTimeRange(timeRange timerange.TimeRange) error {
+	sumDate := timerange.LocDayRange(timeRange.Head).Head //获取此时的零日点
+	//装配渠道统计数据
+	witOrderStateMap := make(WitOrderStateMap)
+	if err := fillWitOrderStatDocMap(sumDate, timeRange, &witOrderStateMap); err != nil {
+		log.Error("withorder RunStat fillWitOrderStatDocMap faild!", log.E(err))
+		return err
+	}
+	//提交渠道统计数据
+	log.Info("withorderStat withdraw committing...", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+	if err := commitWit(sumDate, timeRange.Tail, witOrderStateMap); err != nil {
+		log.Error("withorder RunStat commitWit faild!", log.E(err))
+		return err
+	}
+	log.Info("withorderStat withdraw committed.", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+	return nil
+}
+
+// commit 提交订单统计数据
+func commitWit(sumDate time.Time, recordTime time.Time, stateMap WitOrderStateMap) error {
+	opt := (&db.TransOpts{}).SetReEntry(10)
+	return skdg.StatDB.Trans(func(trans *db.MongoTool) error {
+		if len(stateMap) != 0 {
+			if err := statordermod.ChangeWithDrawStatTrans(trans, sumDate, recordTime, stateMap); err != nil {
+				return fmt.Errorf("stat commitWit ChangeWithDrawStatTrans faild!, err:%+v\n", err)
+			}
+		}
+		//提交本次修改记录
+		if err := statrecordmod.UpsertOneTrans(trans, statrecordmod.WithdrawOrderStatJob, recordTime); err != nil {
+			return fmt.Errorf("stat commitWit UpsertOneTrans faild!, err:%+v\n", err)
+		}
+		return nil
+	}, opt)
+}

skd/job/sync.go

@@ -0,0 +1,74 @@
+package job
+
+import (
+	"91porn-server/skd/job/elastic/esmedia"
+	"91porn-server/skd/job/elastic/esuser"
+	"sync"
+	"time"
+
+	"91porn-server/common"
+	"91porn-server/common/log"
+	"91porn-server/skd/job/elastic/estag"
+	"91porn-server/skd/job/elastic/esvideo"
+	"91porn-server/skd/skdg"
+)
+
+var syncIsRuining = false
+
+// start 定时任务 开启
+func runSync() {
+	//防止重入
+	if syncIsRuining {
+		return
+	}
+	syncIsRuining = true
+	defer func() {
+		syncIsRuining = false
+	}()
+	doSync() //做统计业务
+}
+
+// 做统计业务
+func doSync() {
+	defaultSyncDate, err := getDefaultSyncDate()
+	if err != nil {
+		log.Error("defaultStatDate invalid", log.E(err))
+		return
+	}
+	wg := sync.WaitGroup{}
+	wg.Add(5)
+	// 同步数据
+	{
+		common.Go(func() {
+			defer wg.Done()
+			esuser.RunSync(defaultSyncDate)
+		})
+		common.Go(func() {
+			defer wg.Done()
+			esvideo.RunSync(defaultSyncDate)
+		})
+		common.Go(func() {
+			defer wg.Done()
+			esmedia.RunSync(defaultSyncDate)
+		})
+		common.Go(func() {
+			defer wg.Done()
+			estag.RunSync(defaultSyncDate)
+		})
+
+		common.Go(func() {
+			defer wg.Done()
+			vidDate, err := time.ParseInLocation("2006-01-02", "2023-09-22", time.Local)
+			if err == nil {
+				defaultSyncDate = vidDate
+			}
+			// 同步短视频数据到REDIS
+			//rdvideo.RunSync(defaultSyncDate)
+		})
+	}
+	wg.Wait()
+}
+
+func getDefaultSyncDate() (time.Time, error) {
+	return time.ParseInLocation("2006-01-02", skdg.Conf.StartDate, time.Local)
+}

skd/job/sync_im_users.go

@@ -0,0 +1,176 @@
+package job
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/redis"
+	"91porn-server/skd/skdg"
+	"91porn-server/services/srv_im"
+	"91porn-server/web/webg"
+)
+
+const (
+	phaseQueued           = "queued"
+	phaseScanningDialogs  = "scanning_chat_sessions"
+	phaseRegisteringUsers = "registering_users"
+	phaseAddingFriends    = "adding_friends"
+	phaseDone             = "done"
+
+	syncIMUsersTriggerKey = "im:sync_users:trigger"
+	syncIMUsersLockKey    = "im:sync_users:lock"
+	syncIMUsersStatusKey  = "im:sync_users:status"
+
+	syncIMUsersTriggerTTL = 10 * time.Minute
+	syncIMUsersLockTTL    = 1 * time.Hour
+)
+
+// SyncIMUsersTaskStatus 旧私聊会话用户同步任务状态。
+// 状态完全落在 Redis 里——web 进程写 trigger、读 status；skd 进程消费 trigger、执行任务、写 status。
+type SyncIMUsersTaskStatus struct {
+	Running    bool                        `json:"running"`
+	Phase      string                      `json:"phase,omitempty"`
+	StartedAt  *time.Time                  `json:"startedAt,omitempty"`
+	FinishedAt *time.Time                  `json:"finishedAt,omitempty"`
+	LastStats  srv_im.SyncDialogUsersStats `json:"lastStats"`
+	LastError  string                      `json:"lastError,omitempty"`
+}
+
+// imSyncRedis 拿当前进程可用的 Redis 客户端：
+// skd 主进程用 skdg.Redis；web 主进程用 webg.Redis。
+// 两端写的是同一个 Redis 实例，靠 key 协议互通。
+func imSyncRedis() *redis.Client {
+	if skdg.Redis != nil {
+		return skdg.Redis
+	}
+	if webg.Redis != nil {
+		return webg.Redis
+	}
+	return nil
+}
+
+// RequestSyncIMUsers 由 web admin 接口调用：写 trigger key，由 skd 接管执行。
+// 返回 accepted=true 表示成功排队；false 表示已有任务在跑或在排队。
+func RequestSyncIMUsers() (accepted bool, status SyncIMUsersTaskStatus) {
+	status = GetSyncIMUsersStatus()
+	if status.Running {
+		return false, status
+	}
+	r := imSyncRedis()
+	if r == nil {
+		status.LastError = "redis unavailable"
+		return false, status
+	}
+	if existing, _ := r.Get(syncIMUsersTriggerKey); existing != nil && *existing != "" {
+		// 已经有 trigger 在排队，等 skd 消费
+		return false, status
+	}
+	if err := r.Set(syncIMUsersTriggerKey, "1", syncIMUsersTriggerTTL); err != nil {
+		status.LastError = err.Error()
+		return false, status
+	}
+	now := time.Now()
+	status = SyncIMUsersTaskStatus{
+		Phase:     phaseQueued,
+		StartedAt: &now,
+		LastStats: srv_im.SyncDialogUsersStats{Enabled: true},
+	}
+	saveSyncIMUsersStatus(status)
+	return true, status
+}
+
+// GetSyncIMUsersStatus 读 Redis 里最新状态，任何进程都能调
+func GetSyncIMUsersStatus() SyncIMUsersTaskStatus {
+	r := imSyncRedis()
+	if r == nil {
+		return SyncIMUsersTaskStatus{}
+	}
+	raw, err := r.Get(syncIMUsersStatusKey)
+	if err != nil || raw == nil || *raw == "" {
+		return SyncIMUsersTaskStatus{}
+	}
+	var status SyncIMUsersTaskStatus
+	if err := json.Unmarshal([]byte(*raw), &status); err != nil {
+		return SyncIMUsersTaskStatus{}
+	}
+	return status
+}
+
+// RunPendingSyncIMUsers 由 skd cron 定时调用：检测 trigger 并接管执行同步
+func RunPendingSyncIMUsers() {
+	r := imSyncRedis()
+	if r == nil {
+		return
+	}
+	// 没有 trigger 就直接退
+	trig, _ := r.Get(syncIMUsersTriggerKey)
+	if trig == nil || *trig == "" {
+		return
+	}
+	// 加锁防止 skd 多副本同时跑
+	ok, err := r.Setnx_NewOK(syncIMUsersLockKey, 1, syncIMUsersLockTTL)
+	if err != nil || !ok {
+		return
+	}
+	defer func() { _, _ = r.Del(syncIMUsersLockKey) }()
+	// 消费 trigger，避免下个 tick 重复拉起
+	_, _ = r.Del(syncIMUsersTriggerKey)
+
+	now := time.Now()
+	status := SyncIMUsersTaskStatus{
+		Running:   true,
+		Phase:     phaseScanningDialogs,
+		StartedAt: &now,
+		LastStats: srv_im.SyncDialogUsersStats{Enabled: true},
+	}
+	saveSyncIMUsersStatus(status)
+
+	stats, runErr := runSyncIMUsersOnce(&status)
+
+	finished := time.Now()
+	status.Running = false
+	status.Phase = phaseDone
+	status.FinishedAt = &finished
+	status.LastStats = stats
+	if runErr != nil {
+		status.LastError = runErr.Error()
+		log.Error("RunPendingSyncIMUsers failed", log.E(runErr), log.Any("stats", stats))
+	} else {
+		status.LastError = ""
+		log.Info("RunPendingSyncIMUsers done", log.Any("stats", stats))
+	}
+	saveSyncIMUsersStatus(status)
+}
+
+func runSyncIMUsersOnce(status *SyncIMUsersTaskStatus) (stats srv_im.SyncDialogUsersStats, err error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			err = fmt.Errorf("sync panic: %v", rec)
+		}
+	}()
+	status.Phase = phaseRegisteringUsers
+	saveSyncIMUsersStatus(*status)
+
+	stats, err = srv_im.SyncDialogUsers()
+
+	status.Phase = phaseAddingFriends
+	status.LastStats = stats
+	saveSyncIMUsersStatus(*status)
+	return stats, err
+}
+
+func saveSyncIMUsersStatus(status SyncIMUsersTaskStatus) {
+	r := imSyncRedis()
+	if r == nil {
+		return
+	}
+	raw, err := json.Marshal(status)
+	if err != nil {
+		return
+	}
+	if err := r.Set(syncIMUsersStatusKey, string(raw), 0); err != nil {
+		log.Warn("saveSyncIMUsersStatus set failed", log.E(err))
+	}
+}

skd/job/syncspiderdata/audiobookspider/method.go

@@ -0,0 +1,147 @@
+package audiobookspider
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/httputil"
+	"91porn-server/common/log"
+	"91porn-server/models/v/audioanchormod"
+	"91porn-server/models/v/audiobookmod"
+	"91porn-server/skd/skdg"
+)
+
+func SyncDataFromSpider(data []AudioBook) error {
+	dataLen := len(data)
+	if dataLen == 0 {
+		return nil
+	}
+	now := time.Now()
+	la := make([]audiobookmod.AudioBookUpsertEdit, dataLen)
+	laa := make([]audioanchormod.EditSelector, dataLen)
+	for i := range data {
+		temp := audiobookmod.AudioBookUpsertEdit{
+			Title:     data[i].Title,
+			Anchor:    data[i].Announcer,
+			Cover:     data[i].Cover,
+			FType:     data[i].Category,
+			Summary:   data[i].Desc,
+			Number:    data[i].Id,
+			UpdatedAt: now,
+		}
+		temp.TotalEpisode = len(data[i].EpisodeList)
+		if data[i].Status == 1 {
+			temp.IsActive = true
+		} else {
+			temp.IsActive = false
+		}
+		if data[i].Announcer == "" {
+			temp.IsActive = false
+		}
+		temp.ContentSet = make([]audiobookmod.Content, len(data[i].EpisodeList))
+		for di := range data[i].EpisodeList {
+			temp.ContentSet[di] = audiobookmod.Content{
+				EpisodeNumber:    di + 1,
+				ListenPermission: 0,
+				Name:             data[i].EpisodeList[di].Name,
+				URL:              data[i].EpisodeList[di].FileName,
+				MediaSize:        data[i].EpisodeList[di].Size,
+			}
+		}
+		temp.TrialSet = make([]audiobookmod.Content, len(data[i].TrialEpisodeList))
+		for di := range data[i].TrialEpisodeList {
+			temp.TrialSet[di] = audiobookmod.Content{
+				EpisodeNumber:    di + 1,
+				ListenPermission: 3,
+				Name:             data[i].TrialEpisodeList[di].Name,
+				URL:              data[i].TrialEpisodeList[di].FileName,
+				MediaSize:        data[i].TrialEpisodeList[di].Size,
+			}
+		}
+		tempa := audioanchormod.EditSelector{
+			Name:      data[i].Announcer,
+			Avatar:    data[i].AnnouncerCover,
+			UpdatedAt: now}
+		la[i] = temp
+		laa[i] = tempa
+	}
+
+	log.Info("audiobook sync audiobookSpider data committing...")
+	if err := audiobookmod.InsertBulket(la); err != nil {
+		log.Info("audiobook sync audiobookSpider data commit failed.", log.E(err))
+		return err
+	}
+	if err := audioanchormod.InsertBulket(laa); err != nil {
+		log.Info("audiobook sync audiobookSpider data commit failed.", log.E(err))
+		return err
+	}
+	log.Info("audiobook sync audiobookSpider data committed.")
+	return nil
+}
+
+type Episode struct {
+	Name     string `json:"name" bson:"name" `        //集数名称
+	Size     int64  `json:"size" bson:"size" `        //本集mp3文件大小
+	FileName string `json:"fileName" bson:"fileName"` //小说文件
+}
+
+type AudioBook struct {
+	Id               int64     `json:"id" bson:"_id"`
+	SpiderId         int64     `json:"spiderId" bson:"spiderId"`                 //爬虫库的表Id
+	Timestamp        int64     `json:"timestamp" bson:"timestamp" `              //时间戳
+	Title            string    `json:"title" bson:"title"`                       //标题
+	Announcer        string    `json:"announcer" bson:"announcer"`               //播音员
+	Category         string    `json:"category" bson:"category"`                 //分类
+	Desc             string    `json:"desc" bson:"desc"`                         //描述
+	Cover            string    `json:"cover" bson:"cover" `                      //封面
+	AnnouncerCover   string    `json:"announcerCover"`                           //播音员封面
+	EpisodeList      []Episode `json:"episodeList" bson:"episodeList" `          //集数列表
+	TrialEpisodeList []Episode `json:"trialEpisodeList" bson:"trialEpisodeList"` //试听集数列表
+	Status           int       `json:"status" bson:"status"`                     //状态  1:上架 2:下架
+	CreatedAt        time.Time `json:"createdAt" bson:"createdAt" `              //创建时间
+	UpdatedAt        time.Time `json:"updatedAt" bson:"updatedAt" `              //更新时间
+}
+
+// /api/stat/novel/audio/sync POST
+type SyncAudioBookReq struct {
+	SyncSize  int64     `json:"syncSize" binding:"required"`
+	StartTime time.Time `json:"startTime" binding:"required"`
+}
+
+type SyncAudioBookResp struct {
+	Data []AudioBook `json:"data"`
+}
+
+// todo skd
+// 获取ES同步数据
+func GetListByUpdateTimeRange_net(start time.Time, end time.Time) (data []AudioBook, err error) {
+	p := SyncAudioBookReq{
+		SyncSize:  10000,
+		StartTime: start,
+	}
+	var msg struct {
+		Code int               `json:"code"`
+		Data SyncAudioBookResp `json:"data"`
+	}
+	url := skdg.Conf.Url.SpiderDataURL + "/api/stat/novel/audio/sync"
+	code, err := httputil.DefaultClientPostJsonWithResp(&msg, url, nil, p)
+	if err != nil || code != 200 {
+		log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "POSTJsonWithJResp", err),
+			log.Any("start", start),
+			log.Any("url", end),
+			log.Any("code", code),
+		)
+		return
+	}
+	if msg.Code != 200 {
+		log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
+			log.Any("start", start),
+			log.Any("url", end),
+			log.Any("code", code),
+			log.Any("msg", msg),
+		)
+		return
+	}
+	data = msg.Data.Data
+	return
+}

skd/job/syncspiderdata/audiobookspider/sync.go

@@ -0,0 +1,56 @@
+package audiobookspider
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("Audiobook sync audiobookSpider data start...")
+	defer log.Info("Audiobook sync audiobookSpider data end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SpiderSyncAudiobookJob, defaultDate)
+	if err != nil {
+		log.Error("Audiobook sync audiobookSpider data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now().Add(-time.Minute * 10)
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		data, err := GetListByUpdateTimeRange_net(subTimeRange.Head, subTimeRange.Tail)
+		if err != nil {
+			return
+		}
+		if err = commit(data, subTimeRange.Tail); err != nil {
+			log.Error("Audiobook sync audiobookSpider commit err! ", log.E(err))
+			return
+		}
+		if len(data) > 0 {
+			lastSyncAt = data[len(data)-1].UpdatedAt
+		} else {
+			lastSyncAt = end
+		}
+	}
+}
+
+func commit(data []AudioBook, syncTime time.Time) error {
+	if err := SyncDataFromSpider(data); err != nil {
+		return fmt.Errorf("Audiobook sync audiobookSpider commit audiobook data SyncDataFromSpider failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SpiderSyncAudiobookJob, syncTime); err != nil {
+		return fmt.Errorf("Audiobook sync audiobookSpider commit audiobook data  UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}

skd/job/syncspiderdata/fictionspider/method.go

@@ -0,0 +1,99 @@
+package fictionspider
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/httputil"
+	"91porn-server/common/log"
+	"91porn-server/models/v/fictionmod"
+	"91porn-server/skd/skdg"
+)
+
+func SyncDataFromSpider(data []Novel) error {
+	now := time.Now()
+	if len(data) == 0 {
+		return nil
+	}
+	ll := make([]fictionmod.FictionUpsert, len(data))
+	for i := range data {
+		temp := fictionmod.FictionUpsert{
+			Title:      data[i].Title,
+			FType:      data[i].Type,
+			Summary:    data[i].Desc,
+			ContentURL: data[i].FileName,
+			Number:     data[i].Id,
+			UpdatedAt:  now,
+		}
+		if data[i].Status == 1 {
+			temp.IsActive = true
+		} else {
+			temp.IsActive = false
+		}
+		ll[i] = temp
+	}
+	log.Info("fiction sync fictionSpider data committing...")
+	if err := fictionmod.InsertBulket(ll); err != nil {
+		log.Info("fiction sync fictionSpider data commit failed.")
+		return err
+	}
+	log.Info("fiction sync fictionSpider data committed.")
+	return nil
+}
+
+type Novel struct {
+	Id        int64     `json:"id" bson:"_id"`
+	SpiderId  int64     `json:"spiderId" bson:"spiderId"`    //爬虫库的表Id
+	Title     string    `json:"title" bson:"title"  `        //小说标题
+	Type      string    `json:"type" bson:"type"   `         //小说类型
+	Desc      string    `json:"desc" bson:"desc"  `          //描述
+	FileName  string    `json:"fileName" bson:"fileName"  `  //小说文件
+	Timestamp int64     `json:"timestamp" bson:"timestamp" ` //时间戳
+	Status    int       `json:"status" bson:"status"`        //状态  1:上架 2:下架
+	CreatedAt time.Time `json:"createdAt" bson:"createdAt" ` //创建时间
+	UpdatedAt time.Time `json:"updatedAt" bson:"updatedAt"`  //更新时间
+}
+
+// /api/stat/novel/sync POST
+type SyncNovelReq struct {
+	SyncSize  int64     `json:"syncSize" binding:"required"`
+	StartTime time.Time `json:"startTime" binding:"required"`
+}
+
+type SyncNovelResp struct {
+	Data []Novel `json:"data"`
+}
+
+// todo skd
+// 获取ES同步数据
+func GetListByUpdateTimeRange_net(start time.Time, end time.Time) (data []Novel, err error) {
+	p := SyncNovelReq{
+		SyncSize:  10000,
+		StartTime: start,
+	}
+	var msg struct {
+		Code int           `json:"code"`
+		Data SyncNovelResp `json:"data"`
+	}
+	url := skdg.Conf.Url.SpiderDataURL + "/api/stat/novel/sync"
+	code, err := httputil.DefaultClientPostJsonWithResp(&msg, url, nil, p)
+	if err != nil || code != 200 {
+		log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
+			log.Any("start", start),
+			log.Any("url", end),
+			log.Any("code", code),
+		)
+		return
+	}
+	if msg.Code != 200 {
+		log.Error(fmt.Sprintf("[METHOD-%s]==> %s fail error:%+v:", "GetListByUpdateTimeRange_net", "GetWithJResp", err),
+			log.Any("start", start),
+			log.Any("url", end),
+			log.Any("code", code),
+			log.Any("msg", msg),
+		)
+		return
+	}
+	data = msg.Data.Data
+	return
+}

skd/job/syncspiderdata/fictionspider/sync.go

@@ -0,0 +1,56 @@
+package fictionspider
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("fiction sync fictionSpider data start...")
+	defer log.Info("fiction sync fictionSpider data end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SpiderSyncfictionJob, defaultDate)
+	if err != nil {
+		log.Error("fiction sync fictionSpider data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now().Add(-time.Minute * 10)
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		data, err := GetListByUpdateTimeRange_net(subTimeRange.Head, subTimeRange.Tail)
+		if err != nil {
+			return
+		}
+		if err = commit(data, subTimeRange.Tail); err != nil {
+			log.Error("fiction sync fictionSpider commit err! ", log.E(err))
+			return
+		}
+		if len(data) > 0 {
+			lastSyncAt = data[len(data)-1].UpdatedAt
+		} else {
+			lastSyncAt = end
+		}
+	}
+}
+
+func commit(data []Novel, syncTime time.Time) error {
+	if err := SyncDataFromSpider(data); err != nil {
+		return fmt.Errorf("fiction sync fictionSpider commit fiction data SyncDataFromSpider failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SpiderSyncfictionJob, syncTime); err != nil {
+		return fmt.Errorf("fiction sync fictionSpider commit fiction data  UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}

skd/job/user/privilege.go

@@ -0,0 +1,16 @@
+package user
+
+import (
+	"91porn-server/common/log"
+	"91porn-server/models/v/advanceordermod"
+)
+
+// ResetLimit 重置预售卡预付权益
+func ResetLimit() {
+	err := advanceordermod.ResetToday()
+	if err != nil {
+		log.Error("重置预售卡预付权益错误", log.E(err))
+	}
+
+	log.Info("--重置预售卡预付权益完成！")
+}