forked from all_user/stat_center
Initial commit
This commit is contained in:
@@ -0,0 +1,231 @@
|
||||
package statdistrict
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"stat/common/log"
|
||||
"stat/common/utils"
|
||||
"stat/models/accessmod"
|
||||
"stat/models/channelmod"
|
||||
"stat/models/registermod"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
const (
|
||||
Day1RetainRate string = "day1RetainRate" // 次日留存率
|
||||
Day3RetainRate string = "day3RetainRate" // 3日留存率
|
||||
Day7RetainRate string = "day7RetainRate" // 7日留存率
|
||||
)
|
||||
|
||||
func defaultTimeMap(defaultStatDate time.Time) map[string]time.Time {
|
||||
return map[string]time.Time{
|
||||
Day1RetainRate: defaultStatDate, // 次日留存率
|
||||
Day3RetainRate: defaultStatDate, // 3日留存率
|
||||
Day7RetainRate: defaultStatDate, // 7日留存率
|
||||
}
|
||||
}
|
||||
|
||||
// 获取统计方法
|
||||
type StatMethod = func(ctx context.Context, start time.Time, end time.Time, appIds []int32) error
|
||||
|
||||
// 统计方法汇总
|
||||
var statMethodMap = map[string]StatMethod{
|
||||
Day1RetainRate: Day1RetainRateMethod, // 次日留存率
|
||||
Day3RetainRate: Day3RetainRateMethod, // 3日留存率
|
||||
Day7RetainRate: Day7RetainRateMethod, // 7日留存率
|
||||
}
|
||||
|
||||
// Day1RetainRateMethod 次日留存率
|
||||
func Day1RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
|
||||
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -1))
|
||||
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -1))
|
||||
|
||||
for _, appid := range appIds {
|
||||
districtMap := make(map[string]*registermod.DayRegisterAndVisit)
|
||||
|
||||
// 获取注册用户
|
||||
err := registermod.RegisterUsersInfoByTimeShard(ctx, appid, registerStart, registerEnd, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
|
||||
if value, ok := districtMap[item.DistrictCode]; !ok {
|
||||
districtMap[item.DistrictCode] = ®istermod.DayRegisterAndVisit{
|
||||
RegisterCount: 1,
|
||||
}
|
||||
} else {
|
||||
value.RegisterCount += 1
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("Day1RetainRateMethod RegisterUsersInfoByTimeShard", log.E(err))
|
||||
return err
|
||||
}
|
||||
// 获取访问用户
|
||||
err = accessmod.DailyUserVisitCountByInfoShard(ctx, appid, start, end, utils.GlobalShardHour, registerStart, registerEnd, func(item *accessmod.AccessMod) {
|
||||
if value, ok := districtMap[item.DistrictCode]; !ok {
|
||||
districtMap[item.DistrictCode] = ®istermod.DayRegisterAndVisit{
|
||||
TodayVisitCount: 1,
|
||||
}
|
||||
} else {
|
||||
value.TodayVisitCount += 1
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("Day1RetainRateMethod DailyUserVisitCountByInfoShard", log.E(err))
|
||||
return err
|
||||
}
|
||||
// 插入数据库
|
||||
for districtCode, districtValue := range districtMap {
|
||||
rate := float64(0)
|
||||
if districtValue != nil && districtValue.RegisterCount > 0 {
|
||||
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(districtValue.TodayVisitCount)/float64(districtValue.RegisterCount))), 64)
|
||||
}
|
||||
|
||||
// 更新渠道留存
|
||||
statFilter := bson.M{
|
||||
channelmod.FN.ChannelStat.Channel: districtCode,
|
||||
channelmod.FN.ChannelStat.SumDate: start,
|
||||
channelmod.FN.ChannelStat.AppId: appid,
|
||||
}
|
||||
setDoc := bson.M{
|
||||
"$set": bson.M{
|
||||
channelmod.FN.ChannelStat.Day1RetainRate: rate,
|
||||
},
|
||||
}
|
||||
serr := channelmod.UpdateOneChannelStat(ctx, statFilter, setDoc)
|
||||
if serr != nil {
|
||||
log.Error("Day1RetainRateMethod UpdateOneChannelStat ", log.E(err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Day3RetainRateMethod 3日留存率
|
||||
func Day3RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
|
||||
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -2))
|
||||
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -2))
|
||||
|
||||
for _, appid := range appIds {
|
||||
districtMap := make(map[string]*registermod.DayRegisterAndVisit)
|
||||
// 获取注册用户
|
||||
err := registermod.RegisterUsersInfoByTimeShard(ctx, appid, registerStart, registerEnd, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
|
||||
if value, ok := districtMap[item.DistrictCode]; !ok {
|
||||
districtMap[item.DistrictCode] = ®istermod.DayRegisterAndVisit{
|
||||
RegisterCount: 1,
|
||||
}
|
||||
} else {
|
||||
value.RegisterCount += 1
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("Day3RetainRateMethod RegisterUsersInfoByTimeShard", log.E(err))
|
||||
return err
|
||||
}
|
||||
// 获取访问用户
|
||||
err = accessmod.DailyUserVisitCountByInfoShard(ctx, appid, start, end, utils.GlobalShardHour, registerStart, registerEnd, func(item *accessmod.AccessMod) {
|
||||
if value, ok := districtMap[item.DistrictCode]; !ok {
|
||||
districtMap[item.DistrictCode] = ®istermod.DayRegisterAndVisit{
|
||||
TodayVisitCount: 1,
|
||||
}
|
||||
} else {
|
||||
value.TodayVisitCount += 1
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("Day3RetainRateMethod DailyUserVisitCountByInfoShard", log.E(err))
|
||||
return err
|
||||
}
|
||||
// 插入数据库
|
||||
for districtCode, districtValue := range districtMap {
|
||||
rate := float64(0)
|
||||
if districtValue != nil && districtValue.RegisterCount > 0 {
|
||||
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(districtValue.TodayVisitCount)/float64(districtValue.RegisterCount))), 64)
|
||||
}
|
||||
|
||||
// 更新渠道留存
|
||||
statFilter := bson.M{
|
||||
channelmod.FN.ChannelStat.Channel: districtCode,
|
||||
channelmod.FN.ChannelStat.SumDate: start,
|
||||
channelmod.FN.ChannelStat.AppId: appid,
|
||||
}
|
||||
setDoc := bson.M{
|
||||
"$set": bson.M{
|
||||
channelmod.FN.ChannelStat.Day3RetainRate: rate,
|
||||
},
|
||||
}
|
||||
serr := channelmod.UpdateOneChannelStat(ctx, statFilter, setDoc)
|
||||
if serr != nil {
|
||||
log.Error("Day1RetainRateMethod UpdateOneChannelStat", log.E(err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Day7RetainRateMethod 7日留存率
|
||||
func Day7RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
|
||||
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -6))
|
||||
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -6))
|
||||
for _, appid := range appIds {
|
||||
districtMap := make(map[string]*registermod.DayRegisterAndVisit)
|
||||
// 获取注册用户
|
||||
err := registermod.RegisterUsersInfoByTimeShard(ctx, appid, registerStart, registerEnd, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
|
||||
if value, ok := districtMap[item.DistrictCode]; !ok {
|
||||
districtMap[item.DistrictCode] = ®istermod.DayRegisterAndVisit{
|
||||
RegisterCount: 1,
|
||||
}
|
||||
} else {
|
||||
value.RegisterCount += 1
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("VisitCountMethod GetAccessUserByTime", log.E(err))
|
||||
return err
|
||||
}
|
||||
// 获取访问用户
|
||||
err = accessmod.DailyUserVisitCountByInfoShard(ctx, appid, start, end, utils.GlobalShardHour, registerStart, registerEnd, func(item *accessmod.AccessMod) {
|
||||
if value, ok := districtMap[item.DistrictCode]; !ok {
|
||||
districtMap[item.DistrictCode] = ®istermod.DayRegisterAndVisit{
|
||||
TodayVisitCount: 1,
|
||||
}
|
||||
} else {
|
||||
value.TodayVisitCount += 1
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("VisitCountMethod GetAccessUserByTime", log.E(err))
|
||||
return err
|
||||
}
|
||||
// 插入数据库
|
||||
for districtCode, districtValue := range districtMap {
|
||||
rate := float64(0)
|
||||
if districtValue != nil && districtValue.RegisterCount > 0 {
|
||||
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(districtValue.TodayVisitCount)/float64(districtValue.RegisterCount))), 64)
|
||||
}
|
||||
|
||||
// 更新渠道留存
|
||||
statFilter := bson.M{
|
||||
channelmod.FN.ChannelStat.Channel: districtCode,
|
||||
channelmod.FN.ChannelStat.SumDate: start,
|
||||
channelmod.FN.ChannelStat.AppId: appid,
|
||||
}
|
||||
setDoc := bson.M{
|
||||
"$set": bson.M{
|
||||
channelmod.FN.ChannelStat.Day7RetainRate: rate,
|
||||
},
|
||||
}
|
||||
serr := channelmod.UpdateOneChannelStat(ctx, statFilter, setDoc)
|
||||
if serr != nil {
|
||||
log.Error("Day1RetainRateMethod UpsertOneDistrictStat ", log.E(err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package statdistrict
|
||||
|
||||
import (
|
||||
"context"
|
||||
"stat/common/log"
|
||||
"stat/common/statutils"
|
||||
"stat/common/utils"
|
||||
"stat/models/appInfoMod"
|
||||
statrecordmod "stat/models/statrecord"
|
||||
"time"
|
||||
)
|
||||
|
||||
/*
|
||||
* 主要统计任务:
|
||||
* 1、渠道每天的：直推访问数、裂变访问数、次日留存率、3日留存率、7日留存率。
|
||||
*/
|
||||
func RunStat(defaultStatDate time.Time) error {
|
||||
log.Info("statdistrict stat running...")
|
||||
ctx := context.Background()
|
||||
recordMap := defaultTimeMap(defaultStatDate)
|
||||
|
||||
// 1.获取统计项上次统计的时候(每天执行)
|
||||
recoredResult, err := statrecordmod.LastRecordTimeMap(ctx, statrecordmod.DiscStatJob) //第一次从默认记录时间开始统计
|
||||
if err != nil {
|
||||
log.Error("statdistrict RunStat lastRecordTime failed!", log.E(err))
|
||||
return err
|
||||
}
|
||||
for _, record := range recoredResult {
|
||||
recordMap[record.Item] = record.RecordTime
|
||||
}
|
||||
|
||||
appIds, err := appInfoMod.GetAppIds(ctx)
|
||||
if err != nil {
|
||||
log.Error("statdistrict RunStat GetAppIds failed!", log.E(err))
|
||||
return err
|
||||
}
|
||||
|
||||
statFrq := 24 * time.Hour // 统计频率
|
||||
curStatTime := utils.BeginOfToday() // 本次统计时间
|
||||
|
||||
for item, lastStatAt := range recordMap {
|
||||
_lastStatAt := lastStatAt
|
||||
for curStatTime.After(_lastStatAt) {
|
||||
end := _lastStatAt.Add(statFrq)
|
||||
|
||||
// 超过本次统计结束时间，则结束
|
||||
if end.After(curStatTime) {
|
||||
end = curStatTime
|
||||
}
|
||||
// 根据时间段统计数据
|
||||
err := runStatMethod(ctx, item, _lastStatAt, end, appIds)
|
||||
if err != nil {
|
||||
log.Error("statdistrict statByTimeRange failed!", log.E(err), log.Any("item", item))
|
||||
return err
|
||||
}
|
||||
_lastStatAt = end
|
||||
}
|
||||
recordMap[item] = _lastStatAt
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runStatMethod 调用统计项方法
|
||||
func runStatMethod(ctx context.Context, item string, start, end time.Time, appIds []int32) (err error) {
|
||||
// 1.获取统计方法
|
||||
statMethod, ok := statMethodMap[item]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 调用统计方法，获取数据
|
||||
err = statutils.ServerMon.RunWithName(func() error {
|
||||
return statMethod(ctx, start, end, appIds)
|
||||
}, statutils.GetFuncName(statMethod))
|
||||
|
||||
if err != nil {
|
||||
log.Error("statdistrict stat statByTimeRange statMethod failed!", log.Any("statItem", item), log.E(err))
|
||||
return err
|
||||
}
|
||||
|
||||
// 3.更新记录
|
||||
if err := statrecordmod.UpsertItemRecord(ctx, statrecordmod.DiscStatJob, item, end); err != nil {
|
||||
log.Error("statdistrict stat commit UpsertItemRecord failed!", log.E(err))
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user