root/huangguo_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source 
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
root
2026-09-15 13:57:10 +08:00
co-authored by Claude Opus 5
commit 8679200f41
1897 changed files with 257900 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skd/job/dailyAdverCalc/job.go
+205
View File
@@ -0,0 +1,205 @@
package dailyAdverCalc
import (
"91porn-server/common"
"91porn-server/common/log"
"91porn-server/models/commod"
"91porn-server/models/s/dailyretentionmod"
"91porn-server/models/s/useradverstatmod"
"context"
"fmt"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// retentionKey 用于内存中按 (RegDay, AdGroup) 分组
type retentionKey struct {
RegDay time.Time
AdGroup commod.AdGroup
}
// retentionAccum 内存中的累计数据
type retentionAccum struct {
Retained int64
AdClick int64
TotalClick int64
PayCount int64
PayTotal int64
}
// CalcRetention 计算留存数据
// targetDate: 活跃日期（要统计哪一天的活跃数据）
// 例如：targetDate=15号，会计算所有在15号活跃的用户，按其注册日分组，计算各自的第N天留存
func CalcRetention(ctx context.Context, targetDate time.Time) error {
targetDate = common.NormalizeDate(targetDate)
startDate := targetDate.AddDate(0, 0, -dailyretentionmod.MaxRetentionDays)
accumMap := make(map[retentionKey]*retentionAccum)
// 1. 在 Mongo 中按 (regDay, adGroup) 聚合，避免把当天明细全部拉回内存
stats, err := findUserAdStatSummaryByDate(ctx, targetDate, startDate)
if err != nil {
return fmt.Errorf("query UserAdStat failed: %w", err)
}
for _, stat := range stats {
key := retentionKey{
RegDay: common.NormalizeDate(stat.ID.RegDay),
AdGroup: stat.ID.AdGroup,
}
if accumMap[key] == nil {
accumMap[key] = &retentionAccum{}
}
accumMap[key].Retained = stat.Retained
accumMap[key].AdClick = stat.AdClick
accumMap[key].TotalClick = stat.TotalClick
accumMap[key].PayCount = stat.PayCount
accumMap[key].PayTotal = stat.PayTotal
}
log.Info("calc retention", log.Any("stats.total", len(stats)), log.Any("stats", stats))
// 2. 批量写入 DailyRetention
now := time.Now()
writes := make([]mongo.WriteModel, 0, len(accumMap))
for key, acc := range accumMap {
// 计算是注册后第几天
dayN := int(targetDate.Sub(key.RegDay).Hours() / 24)
if dayN < 0 || dayN > dailyretentionmod.MaxRetentionDays {
log.Info("calc retention check dayN", log.Any("dayN", dayN))
continue // 超出范围，跳过
}
retained := acc.Retained
// 获取已有记录（用于计算留存率和累计值）
existingItem, _ := getDailyRetention(ctx, key.RegDay, key.AdGroup)
cohortSize := retained
if dayN > 0 && existingItem != nil {
cohortSize = existingItem.NewUsers
}
rate := calculateRetentionRate(retained, cohortSize)
// 计算累计值：前一天累计 + 当天值
prevAdClick, prevTotalClick, prevPayCount, prevPayTotal := getPrevAccum(existingItem, dayN)
adClickAcc := prevAdClick + acc.AdClick
totalClickAcc := prevTotalClick + acc.TotalClick
payCountAcc := prevPayCount + acc.PayCount
payTotalAcc := prevPayTotal + acc.PayTotal
writes = append(writes, buildDailyRetentionUpsertModel(
key.RegDay, key.AdGroup, dayN, retained, rate,
adClickAcc, totalClickAcc, payCountAcc, payTotalAcc, now,
))
}
log.Info("calc retention ", log.Any("writes", writes))
if err := dailyretentionmod.BulkWrite(writes, options.BulkWrite().SetOrdered(false)); err != nil {
return fmt.Errorf("bulk upsert daily retention failed: %w", err)
}
return nil
}
// getDailyRetention 获取某个 (RegDay, AdGroup) 的 DailyRetention 记录
func getDailyRetention(ctx context.Context, regDay time.Time, adGroup commod.AdGroup) (*dailyretentionmod.DailyRetention, error) {
filter := bson.M{
"date": regDay,
"adGroup": adGroup,
}
return dailyretentionmod.QueryOne(filter)
}
// buildDailyRetentionUpsertModel 构造留存数据 upsert 模型
func buildDailyRetentionUpsertModel(regDay time.Time, adGroup commod.AdGroup, dayN int, retained int64, rate float64, adClick, totalClick, payCount, payTotal int64, now time.Time) mongo.WriteModel {
filter, update := buildDailyRetentionUpsertData(regDay, adGroup, dayN, retained, rate, adClick, totalClick, payCount, payTotal, now)
return mongo.NewUpdateOneModel().
SetFilter(filter).
SetUpdate(update).
SetUpsert(true)
}
func buildDailyRetentionUpsertData(regDay time.Time, adGroup commod.AdGroup, dayN int, retained int64, rate float64, adClick, totalClick, payCount, payTotal int64, now time.Time) (bson.M, bson.M) {
regDay = common.NormalizeDate(regDay)
filter := bson.M{
"date": regDay,
"adGroup": adGroup,
}
// 使用点号表示法更新 map 中的特定 key
retainedKey := fmt.Sprintf("%s.%d", "userRetained", dayN)
ratesKey := fmt.Sprintf("%s.%d", "userRates", dayN)
adClickKey := fmt.Sprintf("%s.%d", "adClickAcc", dayN)
totalClickKey := fmt.Sprintf("%s.%d", "totalClickAcc", dayN)
payCountKey := fmt.Sprintf("%s.%d", "payCountAcc", dayN)
payTotalKey := fmt.Sprintf("%s.%d", "payTotalAcc", dayN)
update := bson.M{
"$setOnInsert": bson.M{
"date": regDay,
"adGroup": adGroup,
"createdAt": now,
},
"$set": bson.M{
"updatedAt": now,
retainedKey: retained,
ratesKey: rate,
adClickKey: adClick,
totalClickKey: totalClick,
payCountKey: payCount,
payTotalKey: payTotal,
},
}
// 如果是 day0，同时更新 NewUsers
if dayN == 0 {
update["$set"].(bson.M)["newUsers"] = retained
}
return filter, update
}
func calculateRetentionRate(retained, cohortSize int64) float64 {
if cohortSize <= 0 {
return 0
}
return float64(retained) / float64(cohortSize)
}
// getPrevAccum 获取前一天的累计值
func getPrevAccum(item *dailyretentionmod.DailyRetention, dayN int) (adClick, totalClick, payCount, payTotal int64) {
if item == nil || dayN <= 0 {
return 0, 0, 0, 0
}
prevDay := dayN - 1
if item.AdClickAcc != nil {
adClick = item.AdClickAcc[prevDay]
}
if item.TotalClickAcc != nil {
totalClick = item.TotalClickAcc[prevDay]
}
if item.PayCountAcc != nil {
payCount = item.PayCountAcc[prevDay]
}
if item.PayTotalAcc != nil {
payTotal = item.PayTotalAcc[prevDay]
}
return
}
// findDailyRetentionByDateRange 查询指定日期范围内的 DailyRetention 记录
func findDailyRetentionByDateRange(ctx context.Context, startDate, endDate time.Time) ([]*dailyretentionmod.DailyRetention, error) {
filter := bson.M{
"date": bson.M{
"$gte": startDate,
"$lte": endDate,
},
}
return dailyretentionmod.QueryAllList(filter)
}
// findUserAdStatSummaryByDate 查询指定日期在留存窗口内的聚合数据
func findUserAdStatSummaryByDate(ctx context.Context, date, startDate time.Time) ([]*useradverstatmod.DailyRetentionSummary, error) {
local, _ := time.LoadLocation("Asia/Shanghai")
date = time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, local)
startDate = common.NormalizeDate(startDate)
return useradverstatmod.AggregateDailyRetention(date, startDate, date)
}
skd/job/dailyAdverCalc/job_test.go
+105
View File
@@ -0,0 +1,105 @@
package dailyAdverCalc
import (
"91porn-server/common"
"91porn-server/models/commod"
"91porn-server/models/s/dailyretentionmod"
"testing"
"time"
"go.mongodb.org/mongo-driver/bson"
)
func TestCalculateRetentionRate(t *testing.T) {
t.Run("zero cohort size returns zero", func(t *testing.T) {
if got := calculateRetentionRate(10, 0); got != 0 {
t.Fatalf("expected zero rate, got %v", got)
}
})
t.Run("uses retained divided by cohort size", func(t *testing.T) {
got := calculateRetentionRate(25, 100)
want := 0.25
if got != want {
t.Fatalf("expected %v, got %v", want, got)
}
})
}
func TestGetPrevAccum(t *testing.T) {
t.Run("nil item or day zero returns zeros", func(t *testing.T) {
adClick, totalClick, payCount, payTotal := getPrevAccum(nil, 0)
if adClick != 0 || totalClick != 0 || payCount != 0 || payTotal != 0 {
t.Fatalf("expected all zero values, got %d %d %d %d", adClick, totalClick, payCount, payTotal)
}
})
t.Run("reads previous day accumulators", func(t *testing.T) {
item := &dailyretentionmod.DailyRetention{
AdClickAcc: map[int]int64{1: 3, 2: 8},
TotalClickAcc: map[int]int64{1: 5, 2: 13},
PayCountAcc: map[int]int64{1: 1, 2: 2},
PayTotalAcc: map[int]int64{1: 100, 2: 300},
}
adClick, totalClick, payCount, payTotal := getPrevAccum(item, 3)
if adClick != 8 || totalClick != 13 || payCount != 2 || payTotal != 300 {
t.Fatalf("unexpected previous accum values: %d %d %d %d", adClick, totalClick, payCount, payTotal)
}
})
}
func TestBuildDailyRetentionUpsertData(t *testing.T) {
loc, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
t.Fatalf("load location failed: %v", err)
}
regDay := time.Date(2026, 3, 18, 14, 25, 0, 0, loc)
now := time.Date(2026, 3, 19, 9, 30, 0, 0, loc)
t.Run("day zero writes new users and normalizes date", func(t *testing.T) {
filter, update := buildDailyRetentionUpsertData(regDay, commod.AdGroupA, 0, 12, 1, 4, 8, 2, 600, now)
wantDate := common.NormalizeDate(regDay)
if got := filter["date"].(time.Time); !got.Equal(wantDate) {
t.Fatalf("expected normalized date %v, got %v", wantDate, got)
}
if got := filter["adGroup"]; got != commod.AdGroupA {
t.Fatalf("expected ad group %v, got %v", commod.AdGroupA, got)
}
setOnInsert := update["$setOnInsert"].(bson.M)
if got := setOnInsert["date"].(time.Time); !got.Equal(wantDate) {
t.Fatalf("expected setOnInsert date %v, got %v", wantDate, got)
}
if got := setOnInsert["createdAt"].(time.Time); !got.Equal(now) {
t.Fatalf("expected createdAt %v, got %v", now, got)
}
setFields := update["$set"].(bson.M)
if got := setFields["userRetained.0"]; got != int64(12) {
t.Fatalf("expected userRetained.0 to be 12, got %v", got)
}
if got := setFields["userRates.0"]; got != float64(1) {
t.Fatalf("expected userRates.0 to be 1, got %v", got)
}
if got := setFields["newUsers"]; got != int64(12) {
t.Fatalf("expected newUsers to be 12, got %v", got)
}
})
t.Run("non day zero does not overwrite new users", func(t *testing.T) {
_, update := buildDailyRetentionUpsertData(regDay, commod.AdGroupB, 2, 6, 0.3, 9, 11, 3, 800, now)
setFields := update["$set"].(bson.M)
if got := setFields["userRetained.2"]; got != int64(6) {
t.Fatalf("expected userRetained.2 to be 6, got %v", got)
}
if got := setFields["userRates.2"]; got != 0.3 {
t.Fatalf("expected userRates.2 to be 0.3, got %v", got)
}
if _, ok := setFields["newUsers"]; ok {
t.Fatalf("did not expect newUsers on non-day-zero update")
}
})
}