Initial commit

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

skd/job/stat/keyword/method.go

@@ -0,0 +1,59 @@
+package keyword
+
+import (
+	"91porn-server/common/constant"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/l/searchlogmod"
+	"91porn-server/models/s/kwstatmod"
+)
+
+type TimeRange = timerange.TimeRange
+
+func toKeywordIncDoc(realm constant.RealmType, data map[string]int64) []kwstatmod.KeywordIncDoc {
+	list := make([]kwstatmod.KeywordIncDoc, len(data))
+	i := 0
+	for k, v := range data {
+		word := k
+		count := v
+		list[i] = kwstatmod.KeywordIncDoc{
+			Realm: realm,
+			Word:  word,
+			Count: count,
+		}
+		i++
+	}
+	return list
+}
+
+func GetIncDocList(timeRange TimeRange) ([]kwstatmod.KeywordIncDoc, error) {
+	incDocList := []kwstatmod.KeywordIncDoc{}
+	//综合关键字搜索数
+	complexKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchComplex)
+	if err != nil {
+		return nil, err
+	}
+	complexIncDocList := toKeywordIncDoc(constant.SearchComplex, complexKeywordCountMap)
+	incDocList = append(incDocList, complexIncDocList...)
+	//视屏关键字搜索数
+	vidKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchSP)
+	if err != nil {
+		return nil, err
+	}
+	videoIncDocList := toKeywordIncDoc(constant.SearchSP, vidKeywordCountMap)
+	incDocList = append(incDocList, videoIncDocList...)
+	//标签关键字搜索数
+	tagKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchTag)
+	if err != nil {
+		return nil, err
+	}
+	tagIncDocList := toKeywordIncDoc(constant.SearchTag, tagKeywordCountMap)
+	incDocList = append(incDocList, tagIncDocList...)
+	//用户关键字搜索数
+	userKeywordCountMap, err := searchlogmod.GetKeywordCountMapByTime(timeRange.Head, timeRange.Tail, constant.SearchUser)
+	if err != nil {
+		return nil, err
+	}
+	userIncDocList := toKeywordIncDoc(constant.SearchUser, userKeywordCountMap)
+	incDocList = append(incDocList, userIncDocList...)
+	return incDocList, nil
+}

skd/job/stat/keyword/stat.go

@@ -0,0 +1,72 @@
+package keyword
+
+import (
+	"fmt"
+	"time"
+
+	"91porn-server/common/db"
+	"91porn-server/common/log"
+	"91porn-server/common/timeutil/timerange"
+	"91porn-server/models/s/kwstatmod"
+	"91porn-server/models/s/statrecordmod"
+	"91porn-server/skd/skdg"
+)
+
+// RunStat
+func RunStat(defaultStatDate time.Time) {
+	log.Info("keyword stat running...")
+	defer log.Info("keyword stat finished...")
+	now := time.Now()
+	//从历史提交记录中获取上次记录时间
+	lastStatAt, err := statrecordmod.LastRecordTime(statrecordmod.KeywordStatJob, defaultStatDate) //第一次从默认记录时间开始统计
+	if err != nil {
+		log.Error("Keyword RunStat lastRecordTime faild!", log.E(err))
+		return
+	}
+	recentMinute := timerange.RecentMinute(now, statrecordmod.FiveMinuteScale) //对齐本次统计时间 Minute % frequency == 0
+	//拆分时间区间并依次提交,均衡数据库读写压力
+	for recentMinute.After(lastStatAt) {
+		end := lastStatAt.Add(statrecordmod.FiveMinuteScale * time.Minute)
+		//防止时间超出范围
+		if end.After(recentMinute) {
+			end = recentMinute
+		}
+		subTimeRange := timerange.TimeRange{ //连续子切片
+			Head: lastStatAt,
+			Tail: end,
+		}
+		//此时的零日点
+		sumDate := timerange.LocDayRange(subTimeRange.Head).Head
+		//装配基础统计数据
+		incDocList, err := GetIncDocList(subTimeRange)
+		if err != nil {
+			log.Error("Keyword RunStat GetIncDocList faild!", log.E(err))
+			return
+		}
+		//提交数据
+		log.Info("keywordStat committing...", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+		if err = commit(sumDate, subTimeRange.Tail, incDocList); err != nil {
+			log.Error("Keyword RunStat commit faild!", log.E(err))
+			return
+		}
+		log.Info("keywordStat committed.", log.Any("sumDate", sumDate), log.Any("location", sumDate.Location().String()))
+		lastStatAt = end
+	}
+}
+
+// commit 提交统计数据
+func commit(sumDate time.Time, recordTime time.Time, incDocList []kwstatmod.KeywordIncDoc) error {
+	opt := (&db.TransOpts{}).SetReEntry(10)
+	return skdg.StatDB.Trans(func(trans *db.MongoTool) (err error) {
+		if len(incDocList) != 0 {
+			if err = kwstatmod.ChangeStatTrans(trans, sumDate, recordTime, incDocList); err != nil {
+				return fmt.Errorf(" keyword commit ChangeStatTrans faild!, err:%+v\n", err)
+			}
+		}
+		//提交本次修改记录
+		if err = statrecordmod.UpsertOneTrans(trans, statrecordmod.KeywordStatJob, recordTime); err != nil {
+			return fmt.Errorf(" keyword commit UpsertOneTrans faild!, err:%+v\n", err)
+		}
+		return
+	}, opt)
+}