Initial commit

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-15 13:57:10 +08:00
commit 8679200f41
1897 changed files with 257900 additions and 0 deletions
@@ -0,0 +1,27 @@
+package shortvideo
+
+import (
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/log"
+	"91porn-server/common/redis"
+	"91porn-server/models/v/vidmod"
+)
+
+// SyncDataToRedis 同步数据到redis
+func SyncDataToRedis(data []*vidmod.VideoModel) error {
+	if len(data) == 0 {
+		return nil
+	}
+
+	for _, v := range data {
+		if v.NewsType != vidmod.SHORT || (v.Status != vidmod.CheckPass && v.Status != vidmod.Free) {
+			// 去除
+			_, _ = redis.Handler.SRem(redisconst.ShortVideosKey, v.ID.Hex())
+		} else {
+			_, _ = redis.Handler.SAdd(redisconst.ShortVideosKey, v.ID.Hex())
+		}
+	}
+
+	log.Info("Redis sync video data completed.")
+	return nil
+}
@@ -0,0 +1,89 @@
+package shortvideo
+
+import (
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/redis"
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/v/vidmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("Redis short video sync start...")
+	defer log.Info("Redis short video sync end.")
+	// 如果短视频缓存存在的数据量过低，表示缓存被清理过，需要重新同步
+	c, err := redis.Handler.SCard(redisconst.ShortVideosKey)
+	if err == nil && c < 1000 {
+		_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, defaultDate)
+	}
+
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.RedisSyncShortVideoJob, defaultDate)
+	if err != nil {
+		log.Error("Redis sync short video data RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now()
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.OneDayScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.OneDayScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+
+		//// 获取MID
+		//ret, err := moduleconfmod.GetModuleConfByType(moduleconfmod.ShortPage)
+		//if err != nil {
+		//	log.Error("Redis sync short video data RunSync get module conf failed", log.E(err))
+		//	return
+		//}
+		//midMap := make(map[string]any)
+		//for _, v := range ret {
+		//	midMap[v.ID.Hex()] = nil
+		//}
+
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		var page, size = 1, 1000
+		hasNext := true
+		var data []*vidmod.VideoModel
+		for hasNext {
+			data, hasNext, err = vidmod.GetShortVideoListByUpdateTimeRange(subTimeRange.Head, subTimeRange.Tail, page, size)
+			if err != nil {
+				log.Error("Redis sync short video data GetVideoListByUpdateTimeRange err! ", log.E(err))
+				return
+			}
+			if hasNext && len(data) > 0 {
+				_ = SyncDataToRedis(data)
+				page++
+			} else {
+				if err = commit(data, subTimeRange.Tail); err != nil {
+					log.Error("Redis sync short video data RunSync commit err! ", log.E(err))
+					return
+				}
+			}
+		}
+
+		lastSyncAt = end
+	}
+}
+
+func commit(data []*vidmod.VideoModel, syncTime time.Time) error {
+	if len(data) <= 0 {
+		_ = statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime)
+		return nil
+	}
+	if err := SyncDataToRedis(data); err != nil {
+		return fmt.Errorf("Redis sync commit short video data SyncDataToES failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.RedisSyncShortVideoJob, syncTime); err != nil {
+		return fmt.Errorf("Redis sync commit short video data UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}
@@ -0,0 +1,44 @@
+package user
+
+import (
+	"91porn-server/models/v/usermod"
+	"91porn-server/models/v/vidmod"
+	"time"
+)
+
+func doSyncWorkCount(start, end time.Time) error {
+
+	mats := []vidmod.Matcher{
+		(&vidmod.StatusMatch{Status: vidmod.CheckPass}).New(),
+		(&vidmod.UpdatedAtGTEAndLTMatch{GTE: &start, LT: &end}).New(),
+	}
+	publisherIDs, err := vidmod.PublisherIDList(mats...)
+	if err != nil {
+		return err
+	}
+	// 分批操作
+	for len(publisherIDs) > 0 {
+		var uids []uint64
+		if len(publisherIDs) > 100 {
+			uids = publisherIDs[:100]
+			publisherIDs = publisherIDs[100:]
+		} else {
+			uids = publisherIDs
+			publisherIDs = nil
+		}
+
+		mats = []vidmod.Matcher{
+			(&vidmod.StatusMatch{Status: vidmod.CheckPass}).New(),
+			(&vidmod.PublisherIDInMatch{PublisherIDList: uids}).New(),
+		}
+		m, err := vidmod.CountMapByPublisherID(mats...)
+		if err != nil {
+			return err
+		}
+		err = usermod.UpdateUserCountWorks(m)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
@@ -0,0 +1,54 @@
+package user
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("SqlCollasper sync")
+	defer log.Info("SqlCollasper sync user total work  end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SqlCollasperUserTotalWork, defaultDate)
+	if err != nil {
+		log.Error("SqlCollasper RunSync user total work get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now()
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end time.Time
+		if lastSyncAt.Add(time.Hour * 24).Before(now) {
+			end = lastSyncAt.Add(time.Hour * 24)
+		} else {
+			end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+			if end.After(recentMinute) {
+				end = recentMinute
+			}
+		}
+
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		if err = commit(subTimeRange); err != nil {
+			log.Error("SqlCollasper sync audiobook data  RunSync commit err! ", log.E(err))
+			return
+		}
+		lastSyncAt = end
+	}
+}
+
+func commit(t timerange.TimeRange) error {
+	if err := doSyncWorkCount(t.Head, t.Tail); err != nil {
+		return fmt.Errorf("SqlCollasper sync commit incPageView failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SqlCollasperUserTotalWork, t.Tail); err != nil {
+		return fmt.Errorf("SqlCollasper sync commit  UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}
@@ -0,0 +1,51 @@
+package vid
+
+import (
+	"time"
+
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/models/v/vidmod"
+	"91porn-server/skd/skdg"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+func incPageView(syncTime time.Time) error {
+	now := syncTime
+	recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	key := redisconst.VidPageViewListKey(recentMinute.Unix())
+	count, err := skdg.Redis.LCount(key)
+	if err != nil {
+		return err
+	}
+	if count == 0 {
+		return nil
+	}
+	strList, err := skdg.Redis.LRange(key, 0, count-1)
+	if err != nil {
+		return err
+	}
+	dmap := make(map[string]int64)
+	for i := range strList {
+		dmap[strList[i]]++
+	}
+	models := make([]mongo.WriteModel, 0, len(dmap))
+	for k, v := range dmap {
+		_id, err1 := primitive.ObjectIDFromHex(k)
+		if err1 != nil {
+			continue
+		}
+		filter := bson.M{"_id": _id}
+		update := bson.M{"$inc": bson.M{"pageViewCount": v}}
+		models = append(models, mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update))
+	}
+	if err = vidmod.Bulk(models); err != nil {
+		return err
+	}
+	_, _ = skdg.Redis.Del(key)
+	return nil
+}
@@ -0,0 +1,48 @@
+package vid
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/statrecordmod"
+)
+
+func RunSync(defaultDate time.Time) {
+	log.Info("SqlCollasper sync start...")
+	defer log.Info("SqlCollasper sync end.")
+	// 获取上次数据同步的记录时间
+	lastSyncAt, err := statrecordmod.LastRecordTime(statrecordmod.SqlCollasperVidPageView, defaultDate)
+	if err != nil {
+		log.Error("SqlCollasper RunSync get last record time failed", log.E(err))
+		return
+	}
+	var now = time.Now().Add(-time.Minute * 5)
+	var recentMinute = timerange.RecentMinute(now, statrecordmod.FiveMinuteScale)
+	for recentMinute.After(lastSyncAt) {
+		var end = lastSyncAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		var subTimeRange = timerange.TimeRange{
+			Head: lastSyncAt,
+			Tail: end,
+		}
+		if err = commit(subTimeRange.Tail); err != nil {
+			log.Error("SqlCollasper sync audiobook data RunSync commit err! ", log.E(err))
+			return
+		}
+		lastSyncAt = end
+	}
+}
+
+func commit(syncTime time.Time) error {
+	if err := incPageView(syncTime); err != nil {
+		return fmt.Errorf("SqlCollasper sync commit incPageView failed!, err:%+v\n", err)
+	}
+	if err := statrecordmod.UpsertOneTrans(nil, statrecordmod.SqlCollasperVidPageView, syncTime); err != nil {
+		return fmt.Errorf("SqlCollasper sync commit  UpsertOneTrans failed!, err:%+v\n", err)
+	}
+	return nil
+}