all_user/stat_center
3
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
root
2026-05-20 16:13:58 +08:00
commit ceffbe382e
500 changed files with 157469 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skd/job/advertisejob/advertisejob.go
+374
View File
@@ -0,0 +1,374 @@
package advertisejob
import (
"context"
"errors"
"fmt"
"stat/common/config"
"stat/common/log"
"stat/common/redis"
"stat/common/stdLog"
"stat/common/utils"
"stat/models/advertiseLocMod"
"stat/models/advertiseMod"
"stat/models/advertisestat"
"stat/models/appInfoMod"
"stat/models/applicationmod"
"stat/statg"
"strconv"
"strings"
"time"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
)
type TAdsHourCountMap map[int64]advertisestat.THourStat
// AdvertiseClickDayHourStat not include endHour self
func AdvertiseClickDayHourStat(set *redis.Hash, sumDate time.Time, endHour int) error {
data, err := set.NameValues()
if err != nil {
return err
}
if len(data) == 0 {
return nil
}
// app dict
filter := bson.M{}
appDbDict, err := appInfoMod.GetAllAppIdMap(context.Background(), filter)
if err != nil {
return nil
}
source := make(TAdsHourCountMap)
appDict := make(map[int64]int32)
for name, value := range data {
ss := strings.Split(name, redis.AdvertiseClickDayHashIdHourSep)
if len(ss) != 3 {
continue
}
appId, err := strconv.ParseInt(ss[0], 10, 32)
if err != nil {
continue
}
if appId <= 0 {
continue
}
if _, exist := appDbDict[int32(appId)]; !exist {
continue
}
adsId, err := strconv.ParseInt(ss[1], 10, 64)
if err != nil {
continue
}
if adsId <= 0 {
continue
}
hour, err := strconv.Atoi(ss[2])
if err != nil {
continue
}
if hour >= endHour {
continue
}
appDict[adsId] = int32(appId)
stat := source[adsId]
if stat == nil {
stat = make(advertisestat.THourStat)
source[adsId] = stat
for h := 0; h < endHour; h++ {
stat[h] = 0
}
}
if count, err := strconv.ParseInt(value, 10, 64); err == nil {
stat[hour] = count
}
}
ctx := context.Background()
for adsId, stat := range source {
now := time.Now().Local()
total := stat.Total()
update := &advertisestat.Update{
Total: &total,
HourData: &stat,
UpdatedAt: &now,
}
// 尝试更新文档
err = advertisestat.UpdateOneAdvertiseStat(ctx, bson.M{
advertisestat.FN.AdvertiseStat.AdsId: adsId,
advertisestat.FN.AdvertiseStat.SumData: sumDate,
}, bson.M{"$set": update})
if err != nil {
if err == mongo.ErrNoDocuments {
// 不存在则插入
doc := &advertisestat.AdvertiseStat{
Id: primitive.NewObjectID(),
AdsId: adsId,
AppId: appDict[adsId],
SumDate: sumDate,
Total: stat.Total(),
HourData: stat,
UpdatedAt: now,
CreatedAt: now,
}
err = advertisestat.InsertOneAdvertiseStat(ctx, doc)
if err != nil {
return err
}
} else {
return err
}
}
}
return nil
}
func StatAdvertiseHourStat() error {
now := time.Now().Local()
// 给今天的Redis集合设置过期时间
setToday := redis.GetAdvertiseClickDayHash(now)
if err := redis.StatRds.Expire(setToday.Key(), 4*24*time.Hour); err != nil {
log.Error("set redis set expire time failed", log.Any(setToday.Key(), err))
return err
}
// 今日统计
err := AdvertiseClickDayHourStat(setToday, utils.BeginWithTime(now), now.Hour())
if err != nil {
log.Error("AdvertiseClickDayHourStat Today Failed", log.Any(setToday.Key(), err))
return err
}
// 昨日统计
yesterday := now.Add(-24 * time.Hour)
setYesterday := redis.GetAdvertiseClickDayHash(yesterday)
if exists, err := redis.StatRds.Exists(setYesterday.Key()); err == nil && exists {
err = AdvertiseClickDayHourStat(setYesterday, utils.BeginWithTime(yesterday), 24)
if err != nil {
log.Error("AdvertiseClickDayHourStat Yesterday Failed", log.Any(setToday.Key(), err))
return err
}
}
return nil
}
type AdvertiseTemp struct {
Advertise advertiseMod.Advertise `bson:"advertise"` // 应用id
Loc advertiseLocMod.AdvertiseLocation `bson:"loc"` // 应用id
}
type AdvertiseMap struct {
AppId int32 `bson:"appId"` // 应用id
Name string `bson:"name"` // 应用name
List []AdvertiseTemp `bson:"list"` // 广告list
}
type ApplicationMap struct {
AppId int32 `bson:"appId"` // 所属产品appIdid
Name string `bson:"name"` // 所属产品name
List []ApplicationTemp `bson:"list"` // list
}
type ApplicationTemp struct {
Application applicationmod.Application `bson:"advertise"` // 应用信息
}
func AdvertiseExpireSend() error {
ctx := context.Background()
now := time.Now()
tomorrow := now.AddDate(0, 0, 1)
sort := bson.D{{Key: "appId", Value: 1}}
opts := options.Find().SetSort(sort)
filter := bson.M{"isActive": true,
"endTime": bson.M{"$lte": tomorrow}}
list, serr := advertiseMod.FindAdvertiselist(ctx, filter, opts)
if serr != nil && serr.Code != stdLog.DataEmpty {
log.Error("AdvertiseExpireSend err", log.E(serr))
return errors.New(serr.Error())
}
if len(list) <= 0 {
return nil
}
// 获取app信息
appIds := make([]int32, 0)
for _, v := range list {
appIds = append(appIds, v.AppId)
}
appList, err := appInfoMod.GetAppInfosByAppIds(ctx, appIds)
if err != nil {
log.Error("AdvertiseExpireSend GetAppInfosByAppIds err", log.E(err))
return err
}
appMap := make(map[int32]appInfoMod.AppInfo)
for _, v := range appList {
appMap[v.AppId] = v
}
adsmap := make(map[int32]*AdvertiseMap)
for _, v := range list {
// 获取位置信息
adsLocInfo, err := advertiseLocMod.FindLocationByAppIdAndLocId(ctx, v.AppId, v.LocId)
if err != nil {
log.Error("AdvertiseExpireSend FindLocationByAppIdAndLocId err", log.E(err),
log.Any("appid", v.AppId), log.Any("locid", v.LocId))
continue
}
if info, ok := adsmap[v.AppId]; ok {
temp := AdvertiseTemp{
Advertise: v,
Loc: adsLocInfo,
}
info.List = append(info.List, temp)
} else {
ads := &AdvertiseMap{}
ads.List = make([]AdvertiseTemp, 0)
ads.AppId = v.AppId
if appInfo, ok := appMap[v.AppId]; ok {
ads.Name = appInfo.Name
}
temp := AdvertiseTemp{
Advertise: v,
Loc: adsLocInfo,
}
ads.List = append(ads.List, temp)
adsmap[ads.AppId] = ads
}
}
if statg.Bot != nil {
text := ""
for _, info := range adsmap {
text += "=======================================\n"
text += fmt.Sprintf("【%s】有广告已过期\n", info.Name)
for _, value := range info.List {
if value.Advertise.EndTime.After(now) {
text += fmt.Sprintf(" 【标题: %s】【位置: %s】将过期 【%s】 \n",
value.Advertise.Title, value.Loc.Name, value.Advertise.EndTime.Format("2006.01.02"))
} else {
text += fmt.Sprintf(" 【标题: %s】【位置: %s】已过期 【%s】 \n",
value.Advertise.Title, value.Loc.Name, value.Advertise.EndTime.Format("2006.01.02"))
}
}
}
if text != "" {
text += statg.Tgbotsend
statg.Bot.SendAdvsSendMessage(text)
}
} else {
log.Error("AdvertiseExpireSend statg.Bot nil")
}
return nil
}
// 过期应用检测告警
func ApplicationExpireSend() error {
ctx := context.Background()
now := time.Now()
tomorrow := now.AddDate(0, 0, 1)
sort := bson.D{{Key: "endTime", Value: 1}}
opts := options.Find().SetSort(sort)
filter := bson.M{"status": true,
"endTime": bson.M{"$lte": tomorrow}}
list, serr := applicationmod.FindManyApplication(ctx, filter, opts)
if serr != nil && serr.Code != stdLog.DataEmpty {
log.Error("AdvertiseExpireSend err", log.E(serr))
return errors.New(serr.Error())
}
if len(list) <= 0 {
return nil
}
// 获取app信息
appIds := make([]int32, 0)
for _, v := range list {
appIds = append(appIds, int32(v.ParentAppId))
}
appList, err := appInfoMod.GetAppInfosByAppIds(ctx, appIds)
if err != nil {
log.Error("AdvertiseExpireSend GetAppInfosByAppIds err", log.E(err))
return err
}
appMap := make(map[int32]appInfoMod.AppInfo)
for _, v := range appList {
appMap[v.AppId] = v
}
appsmap := make(map[int32]*ApplicationMap)
for _, v := range list {
if info, ok := appsmap[int32(v.ParentAppId)]; ok {
temp := ApplicationTemp{
Application: *v,
}
info.List = append(info.List, temp)
} else {
apps := &ApplicationMap{}
apps.List = make([]ApplicationTemp, 0)
apps.AppId = int32(v.ParentAppId)
if appInfo, ok := appMap[int32(v.ParentAppId)]; ok {
apps.Name = appInfo.Name
}
temp := ApplicationTemp{
Application: *v,
}
apps.List = append(apps.List, temp)
appsmap[apps.AppId] = apps
}
}
if statg.Bot != nil {
text := ""
for _, info := range appsmap {
text += "=======================================\n"
text += fmt.Sprintf("【%s】有应用过期告警\n", info.Name)
for _, value := range info.List {
if value.Application.EndTime.After(now) {
text += fmt.Sprintf(" 【应用名称: %s】【ID: %s】将过期 【%s】 \n",
value.Application.Name, value.Application.Id.Hex(), value.Application.EndTime.Format("2006.01.02"))
} else {
text += fmt.Sprintf(" 【应用名称: %s】【ID: %s】已过期 【%s】 \n",
value.Application.Name, value.Application.Id.Hex(), value.Application.EndTime.Format("2006.01.02"))
}
}
}
if text != "" {
text += statg.Tgbotsend
statg.Bot.SendAdvsSendMessage(text)
}
} else {
log.Error("ApplicationExpireSend statg.Bot nil")
}
return nil
}
func LoadTgbotsend() error {
conf := statg.GlobalConfig{}
err := config.LoadJSON("conf/server.json", &conf)
if err != nil {
log.Fatal("load配置失败", log.E(err))
return err
}
statg.Tgbotsend = conf.TgbotSend
fmt.Println(statg.Tgbotsend)
return nil
}
skd/job/application/application.go
+195
View File
@@ -0,0 +1,195 @@
package applicationjob
import (
"context"
"stat/common/log"
"stat/common/redis"
"stat/common/stdLog"
"stat/common/utils"
"stat/models/applicationmod"
"stat/models/applicationstat"
"stat/service/innerserverser"
"strconv"
"strings"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type TAppHourCountMap map[string]applicationstat.THourStat
// AppClickDayHourStat not include endHour self
func AppClickDayHourStat(set *redis.Hash, sumDate time.Time, endHour int) error {
data, err := set.NameValues()
if err != nil {
return err
}
if len(data) == 0 {
return nil
}
source := make(TAppHourCountMap)
for name, value := range data {
ss := strings.Split(name, redis.AppClickDaySetIdHourSep)
if len(ss) != 2 {
continue
}
id := ss[0]
if len(id) != 24 {
continue
}
hour, err := strconv.Atoi(ss[1])
if err != nil {
continue
}
if hour >= endHour {
continue
}
stat := source[id]
if stat == nil {
stat = make(applicationstat.THourStat)
source[id] = stat
for h := 0; h < endHour; h++ {
stat[h] = 0
}
}
if count, err := strconv.ParseInt(value, 10, 64); err == nil {
stat[hour] = count
}
}
ctx := context.Background()
for id, stat := range source {
/*appidslice := strings.Split(id, "_")
if len(appidslice) != 2 {
log.Error("AppClickDayHourStat", log.Any(id, stat))
continue
}
appId, err := strconv.Atoi(appidslice[0])
if err != nil {
continue
}
parentAppId, err := strconv.Atoi(appidslice[1])
if err != nil {
continue
}
*/
appId, err := primitive.ObjectIDFromHex(id)
if err != nil {
continue
}
now := time.Now().Local()
total := stat.Total()
update := &applicationstat.Update{
Total: &total,
HourData: &stat,
UpdatedAt: &now,
}
// 尝试更新文档
sErr := applicationstat.UpdateOneApplicationStat(ctx, bson.M{
"applicationDbId": appId,
applicationstat.FN.ApplicationStat.SumData: sumDate,
}, bson.M{"$set": update})
if sErr != nil {
if sErr.Code == stdLog.DataNotExist {
// 不存在则插入
doc := &applicationstat.ApplicationStat{
Id: primitive.NewObjectID(),
ApplicationDbId: appId,
SumDate: sumDate,
Total: stat.Total(),
HourData: stat,
UpdatedAt: now,
CreatedAt: now,
}
sErr = applicationstat.InsertOneApplicationStat(ctx, doc)
if sErr != nil {
return sErr
}
} else {
return sErr
}
}
}
return nil
}
func StatAppHourClick() error {
log.Info("configser:StatAppHourClick begin")
defer log.Info("configser:StatAppHourClick end")
now := time.Now().Local()
// 给今天的Redis集合设置过期时间
setToday := redis.GetAppClickDaySet(now)
if err := redis.StatRds.Expire(setToday.Key(), 2*24*time.Hour); err != nil {
log.Error("set redis set expire time failed", log.Any(setToday.Key(), err))
return err
}
// 今日统计
err := AppClickDayHourStat(setToday, utils.BeginWithTime(now), now.Hour())
if err != nil {
log.Error("AppClickDayHourStat Today Failed", log.Any(setToday.Key(), err))
return err
}
// 昨日统计
yesterday := now.Add(-24 * time.Hour)
setYesterday := redis.GetAppClickDaySet(yesterday)
if exists, err := redis.StatRds.Exists(setYesterday.Key()); err == nil && exists {
err = AppClickDayHourStat(setYesterday, utils.BeginWithTime(yesterday), 24)
if err != nil {
log.Error("AppClickDayHourStat Yesterday Failed", log.Any(setToday.Key(), err))
return err
}
// 删除昨日统计集合
redis.StatRds.Del(setYesterday.Key())
}
return nil
}
func RechargeDayXlsl() error {
err := innerserverser.InnerServerDailyStat(context.Background())
if err != nil {
log.Error("RechargeDayXlsl", log.Any("err", err))
return err
}
return nil
}
func RechargeMonthXlsl() error {
err := innerserverser.InnerServerMonthStat2(context.Background(), nil, nil, nil)
if err != nil {
log.Error("RechargeMonthXlsl", log.Any("err", err))
return err
}
return nil
}
func CloseExpiredApplications() error {
log.Info("applicationjob:CloseExpiredApplications begin")
defer log.Info("applicationjob:CloseExpiredApplications end")
now := time.Now().Local()
filter := bson.M{
"status": true,
"endTime": bson.M{"$lt": now},
}
update := bson.M{"$set": bson.M{"status": false}}
serr := applicationmod.UpdateManyApplication(context.Background(), filter, update)
if serr != nil {
log.Error("application FindAdsByAppId Find err", log.E(serr))
return serr
}
return nil
}
skd/job/blocstatdaily/method.go
+450
View File
@@ -0,0 +1,450 @@
package blocstatdaily
import (
"context"
"fmt"
"stat/common/log"
"stat/common/utils"
"stat/models/accessmod"
"stat/models/blocDailystatmod"
"stat/models/rechargemod"
"stat/models/registermod"
"stat/models/withdrawUserOrdermod"
"strconv"
"time"
)
const (
DirectVisitCount string = "directVisitCount" // 直推访问数
FissionVisitCount string = "fissionVisitCount" // 裂变访问数
Day1RetainRate string = "day1RetainRate" // 次日留存率
Day3RetainRate string = "day3RetainRate" // 3日留存率
Day7RetainRate string = "day7RetainRate" // 7日留存率
RechargeAmount string = "rechargeAmount" // 直推充值金额
FissionAmount string = "fissionAmount" // 裂变充值金额
NewDirectCount string = "newDirectCount" // 新增直推
NewFissionCount string = "newFissionCount" // 新增裂变
DaichongAmout string = "daichongAmout" // 代充金额
OnlineAmount string = "onlineAmount" // 在线充值
NewRechargeCount string = "newRechargeCount" // 新增充值数
UserWithdrawAmount string = "userWithdrawAmount" // 用户提现
)
func defaultTimeMap(defaultStatDate time.Time) map[string]time.Time {
return map[string]time.Time{
DirectVisitCount: defaultStatDate, // 直推访问用户数
FissionVisitCount: defaultStatDate, // 裂变访问数
Day1RetainRate: defaultStatDate, // 次日留存率
Day3RetainRate: defaultStatDate, // 3日留存率
Day7RetainRate: defaultStatDate, // 7日留存率
RechargeAmount: defaultStatDate, // 直推充值金额
FissionAmount: defaultStatDate, // 裂变充值金额
NewDirectCount: defaultStatDate, // 新增直推
NewFissionCount: defaultStatDate, // 新增裂变
DaichongAmout: defaultStatDate, // 代充金额
OnlineAmount: defaultStatDate, // 在线充值
NewRechargeCount: defaultStatDate, // 新增充值数
UserWithdrawAmount: defaultStatDate, // 用户提现
}
}
// 获取统计方法
type StatMethod = func(ctx context.Context, start time.Time, end time.Time, appIds []int32) error
// 统计方法汇总
var statMethodMap = map[string]StatMethod{
DirectVisitCount: DirectVisitCountMethod, // 直推访问用户数
FissionVisitCount: FissionVisitCountMethod, // 裂变访问数
Day1RetainRate: Day1RetainRateMethod, // 次日留存率
Day3RetainRate: Day3RetainRateMethod, // 3日留存率
Day7RetainRate: Day7RetainRateMethod, // 7日留存率
RechargeAmount: DirectAmountMethod, // 直推充值金额
FissionAmount: FissionAmountMethod, // 裂变充值金额
NewDirectCount: NewDirectCountMethod, // 新增直推
NewFissionCount: NewFissionCountMethod, // 新增裂变
DaichongAmout: DaichongAmoutMethod, // 代充金额
OnlineAmount: OnlineAmountMethod, // 在线充值
NewRechargeCount: NewRechargeCountMethod, // 新增充值数
UserWithdrawAmount: UserWithdrawAmountMethod, // 用户提现
}
// DirectVisitCountMethod 访问用户数
func DirectVisitCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
isDirect := true
userCount, err := accessmod.BlocDailyStatCount(ctx, appid, &isDirect, nil, start, end)
if err != nil {
log.Error("DirectVisitCountMethod BlocDailyStatCount", log.E(err))
return err
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
DirectVisitCount: &userCount,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("DirectVisitCountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// FissionVisitCountMethod 裂变访问数
func FissionVisitCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
isDirect := false
userCount, err := accessmod.BlocDailyStatCount(ctx, appid, &isDirect, nil, start, end)
if err != nil {
log.Error("DirectVisitCountMethod BlocDailyStatCount", log.E(err))
return err
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
FissionVisitCount: &userCount,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("FissionVisitCountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// Day1RetainRateMethod 次日留存率
func Day1RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
rate := float64(0)
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -1))
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -1))
registerCount, err := registermod.TotalAddUserCount(ctx, appid, registerStart, registerEnd)
if err != nil {
log.Error("Day1RetainRateMethod s", log.E(err))
return err
}
VistitCount, err := accessmod.DailyUserVisitCountByRegister(ctx, appid, start, end, registerStart, registerEnd)
if err != nil {
log.Error("Day1RetainRateMethod DailyUserVisitCountByRegister", log.E(err))
return err
}
if registerCount > 0 {
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(VistitCount)/float64(registerCount))), 64)
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
Day1RetainRate: &rate,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("Day1RetainRateMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// Day3RetainRateMethod 3日留存率
func Day3RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -2))
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -2))
rate := float64(0)
registerCount, err := registermod.TotalAddUserCount(ctx, appid, registerStart, registerEnd)
if err != nil {
log.Error("Day3RetainRateMethod DailyUserRegisterCount", log.E(err))
return err
}
VistitCount, err := accessmod.DailyUserVisitCountByRegister(ctx, appid, start, end, registerStart, registerEnd)
if err != nil {
log.Error("Day3RetainRateMethod DailyUserVisitCountByRegister", log.E(err))
return err
}
if registerCount > 0 {
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(VistitCount)/float64(registerCount))), 64)
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
Day3RetainRate: &rate,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("Day3RetainRateMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// Day7RetainRateMethod 7日留存率
func Day7RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -6))
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -6))
rate := float64(0)
registerCount, err := registermod.TotalAddUserCount(ctx, appid, registerStart, registerEnd)
if err != nil {
log.Error("Day3RetainRateMethod DailyUserRegisterCount", log.E(err))
return err
}
VistitCount, err := accessmod.DailyUserVisitCountByRegister(ctx, appid, start, end, registerStart, registerEnd)
if err != nil {
log.Error("Day3RetainRateMethod DailyUserVisitCountByRegister", log.E(err))
return err
}
if registerCount > 0 {
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(VistitCount)/float64(registerCount))), 64)
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
Day7RetainRate: &rate,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("Day7RetainRateMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// DirectAmountMethod 直推充值金额
func DirectAmountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
// 获取充值金额
count, amount, err := rechargemod.BlocDirectRechargeAmount(ctx, appid, start, end)
if err != nil {
log.Error("DirectAmountMethod BlocDirectRechargeAmount ", log.E(err))
return err
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
DirectAmount: &amount,
DirectAmountCount: &count,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("DirectAmountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// FissionAmountMethod 裂变充值金额
func FissionAmountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
// 获取充值金额
count, amount, err := rechargemod.BlocFissionRechargeAmount(ctx, appid, start, end)
if err != nil {
log.Error("FissionAmountMethod BlocFissionRechargeAmount ", log.E(err))
return err
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
FissionAmount: &amount,
FissionAmountCount: &count,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("FissionAmountMethod UpsertBlocDailyStat ", log.E(err))
return err
}
}
return nil
}
// NewDirectCountMethod 新增直推
func NewDirectCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
count, err := registermod.BlocDirectAddUserCount(ctx, appid, start, end)
if err != nil {
log.Error("NewDirectCountMethod DeductionRechargeAmount ", log.E(err))
return err
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
NewDirectCount: &count,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("NewDirectCountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// NewFissionCountMethod 新增裂变
func NewFissionCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
count, err := registermod.BlocFissionAddUserCount(ctx, appid, start, end)
if err != nil {
log.Error("NewFissionCountMethod DeductionRechargeAmount ", log.E(err))
return err
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
NewFissionCount: &count,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("NewFissionCountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// DaichongAmoutMethod 代充金额
func DaichongAmoutMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
_, money, err := rechargemod.BlocGetDaichongAmout(ctx, appid, start, end)
if err != nil {
log.Error("DaichongAmoutMethod BlocGetDaichongAmout ", log.E(err))
return err
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
DaichongAmout: &money,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("NewDeductionCountMethod UpsertBlocDailyStat ", log.E(err))
return err
}
}
return nil
}
// OnlineAmountMethod 在线充值
func OnlineAmountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
_, money, err := rechargemod.BlocGetOnlineAmount(ctx, appid, start, end)
if err != nil {
log.Error("DaichongAmoutMethod BlocGetOnlineAmount ", log.E(err))
return err
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
OnlineAmount: &money,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("NewDeductionCountMethod UpsertBlocDailyStat ", log.E(err))
return err
}
}
return nil
}
// NewRechargeCountMethod 新增充值
func NewRechargeCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
// 获取充值金额
result, err := rechargemod.BlocTotalDistricRechargeAmount(ctx, appid, start, end)
if err != nil {
log.Error("NewRechargeCountMethod BlocTotalDistricRechargeAmount ", log.E(err))
return err
}
totalRechargeCount := int64(len(result))
amountMap := make(map[uint64]int64)
userIds := make([]uint64, 0)
for _, value := range result {
userIds = append(userIds, value.UserId)
if key, ok := amountMap[value.UserId]; ok {
//存在
amountMap[value.UserId] = value.PayMoney + key
} else {
amountMap[value.UserId] = value.PayMoney
}
}
// 获取注册信息
userList, err := registermod.TimeVisitUsersInfoByTime(ctx, appid, userIds, start, end)
if err != nil {
log.Error("NewRechargeCountMethod TotalDistricRechargeAmount ", log.E(err))
return err
}
amount := int64(0)
count := int64(0)
for _, user := range userList {
count += 1
if key, ok := amountMap[user.UserId]; ok {
amount += key
}
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
NewRechargeCount: &count,
NewRechargeAmount: &amount,
TotalRechargeCount: &totalRechargeCount,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("DeductionAmountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// UserWithdrawAmountMethod 用户提现
func UserWithdrawAmountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
// 获取充值金额
result, err := withdrawUserOrdermod.GetBlocUserWithdrawOrder(ctx, appid, start, end)
if err != nil {
log.Error("NewRechargeCountMethod BlocTotalDistricRechargeAmount ", log.E(err))
return err
}
payMoney := int64(0)
money := int64(0)
for _, withdrawInfo := range result {
payMoney += withdrawInfo.PayMoney
money += withdrawInfo.Money
}
// 更新数据库
doc := blocDailystatmod.BlocDailyStatDoc{
UserWithDrawAmount: &money,
UserWithDrawPaymoneyAmount: &payMoney,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("DeductionAmountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
skd/job/blocstatdaily/stat.go
+112
View File
@@ -0,0 +1,112 @@
package blocstatdaily
import (
"context"
"stat/common/log"
"stat/common/statutils"
"stat/common/utils"
"stat/models/appInfoMod"
statrecordmod "stat/models/statrecord"
"time"
)
/*
* 主要统计任务:
* 1、统计每天的app的直推访问数、裂变访问数、次日留存率、3日留存率、7日留存率。（表格）
* 2、按5分钟一个点，记录实时数据 app（新增、访问数....）。（曲线图）
*/
// RunDayStat 任务1:统计前一天日活（每天1点开始统计）
func RunStat(defaultStatDate time.Time) error {
log.Info("daily stat running...")
ctx := context.Background()
recordMap := defaultTimeMap(defaultStatDate)
// 1.获取统计项上次统计的时候(每天执行)
recoredResult, err := statrecordmod.LastRecordTimeMap(ctx, statrecordmod.BlocDailyStatDayJob) //第一次从默认记录时间开始统计
if err != nil {
log.Error("Daily RunStat lastRecordTime failed!", log.E(err))
return err
}
for _, record := range recoredResult {
recordMap[record.Item] = record.RecordTime
}
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("daily RunStat GetAppIds failed!", log.E(err))
return err
}
statFrq := 24 * time.Hour // 统计频率
curStatTime := utils.BeginOfToday() // 本次统计时间
for item, lastStatAt := range recordMap {
_lastStatAt := lastStatAt
for curStatTime.After(_lastStatAt) {
end := _lastStatAt.Add(statFrq)
// 超过本次统计结束时间，则结束
if end.After(curStatTime) {
end = curStatTime
}
err := runStatMethod(ctx, item, _lastStatAt, end, appIds)
if err != nil {
log.Error("daily statByTimeRange failed!", log.E(err), log.Any("item", item))
return err
}
_lastStatAt = end
}
recordMap[item] = _lastStatAt
}
return nil
}
// 实时统计每天的情况
func RunStatToday() {
ctx := context.Background()
start := utils.BeginOfToday()
end := utils.EndOfToday()
log.Info("daily today stat running...")
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("daily today RunStat GetAppIds failed!", log.E(err))
return
}
for item, statMethod := range statMethodMap {
// 调用统计方法，获取数据
err := statMethod(ctx, start, end, appIds)
if err != nil {
log.Error("Daily today stat statByTimeRange statMethod failed!", log.Any("statItem", item), log.E(err))
}
}
}
// runStatMethod 调用统计项方法
func runStatMethod(ctx context.Context, item string, start, end time.Time, appIds []int32) (err error) {
// 1.获取统计方法
statMethod, ok := statMethodMap[item]
if !ok {
return nil
}
// 调用统计方法，获取数据
err = statutils.ServerMon.RunWithName(func() error {
return statMethod(ctx, start, end, appIds)
}, statutils.GetFuncName(statMethod))
if err != nil {
log.Error("Daily stat statByTimeRange statMethod failed!", log.Any("statItem", item), log.E(err))
return err
}
// 3.更新记录
if err := statrecordmod.UpsertItemRecord(ctx, statrecordmod.BlocDailyStatDayJob, item, end); err != nil {
log.Error("Daily stat commit UpsertItemRecord failed!", log.E(err))
return err
}
return nil
}
skd/job/blocstatdistrict/method.go
+197
View File
@@ -0,0 +1,197 @@
package blocstatdistrict
import (
"context"
"stat/common/log"
"stat/common/utils"
"stat/models/accessmod"
"stat/models/channelCodeBlocStatMod"
"stat/models/rechargemod"
"stat/models/registermod"
"time"
"go.mongodb.org/mongo-driver/bson"
)
const (
DirectVisitCount string = "directVisitCount" // 访问数
RechargeAmount string = "rechargeAmount" // 充值金额
NewDirectCount string = "newDirectCount" // 新增
)
func defaultTimeMap(defaultStatDate time.Time) map[string]time.Time {
return map[string]time.Time{
DirectVisitCount: defaultStatDate, // 访问数
RechargeAmount: defaultStatDate, // 充值金额
NewDirectCount: defaultStatDate, // 新增
}
}
// 获取统计方法
type StatMethod = func(ctx context.Context, start time.Time, end time.Time, appIds []int32) error
// 统计方法汇总
var statMethodMap = map[string]StatMethod{
DirectVisitCount: DirectVisitCountMethod, // 访问数
RechargeAmount: RechargeAmountMethod, // 充值金额
NewDirectCount: NewDirectCountMethod, // 新增
}
// DirectVisitCountMethod 直推访问数
func DirectVisitCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
type districtVisitInfo struct {
DirectVisitCount int64 // 直推访问数
FissionVisitCount int64 // 裂变访问数
}
for _, appid := range appIds {
districtVisitMap := make(map[string]*districtVisitInfo)
err := accessmod.VisitInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *accessmod.AccessMod) {
if len(item.DistrictCode) > 0 && !item.BlocDeduction {
if item.IsDirect {
if value, ok := districtVisitMap[item.DistrictCode]; !ok {
districtVisitMap[item.DistrictCode] = &districtVisitInfo{
DirectVisitCount: 1,
}
} else {
value.DirectVisitCount += 1
}
} else if !item.IsDirect { // 裂变
if value, ok := districtVisitMap[item.DistrictCode]; !ok {
districtVisitMap[item.DistrictCode] = &districtVisitInfo{
FissionVisitCount: 1,
}
} else {
value.FissionVisitCount += 1
}
}
}
})
if err != nil {
log.Error("VisitCountMethod VisitInfoByTimeShard ", log.E(err))
return err
}
// 插入数据库
for k, v := range districtVisitMap {
// 更新渠道存
err = channelCodeBlocStatMod.UpsertOneDistrictVisitStat(ctx, k, appid, start, v.DirectVisitCount, v.FissionVisitCount)
if err != nil {
log.Error("VisitCountMethod UpsertOneDistrictVisitStat ", log.E(err))
return err
}
}
}
return nil
}
// RechargeAmountMethod 用户充值
func RechargeAmountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
type districtRechargeUserIds struct {
RechangeAmount int64
FissionAmount int64
DeductionAmount int64
}
for _, appid := range appIds {
rechangeUserMap := make(map[string]*districtRechargeUserIds)
// 获取充值金额
filter := bson.M{"appId": appid, "successAt": bson.M{"$gte": start, "$lt": end}, "districtCode": bson.M{"$gt": ""}}
rechargeList, _ := rechargemod.FindManyRechargeInfo(ctx, filter)
for _, rechargeInfo := range rechargeList {
if len(rechargeInfo.DistrictCode) > 0 { // 充值不存在扣量
if rechargeInfo.IsDirect {
if value, ok := rechangeUserMap[rechargeInfo.DistrictCode]; !ok {
rechangeUserMap[rechargeInfo.DistrictCode] = &districtRechargeUserIds{
RechangeAmount: rechargeInfo.Money,
}
} else {
value.RechangeAmount += rechargeInfo.Money
}
} else {
if value, ok := rechangeUserMap[rechargeInfo.DistrictCode]; !ok {
rechangeUserMap[rechargeInfo.DistrictCode] = &districtRechargeUserIds{
FissionAmount: rechargeInfo.Money,
}
} else {
value.FissionAmount += rechargeInfo.Money
}
}
}
}
// 插入数据库
for k, v := range rechangeUserMap {
// 更新渠道存
err := channelCodeBlocStatMod.UpsertOneDistrictRechargeStat(ctx, k, appid, start, v.RechangeAmount, v.FissionAmount, v.DeductionAmount)
if err != nil {
log.Error("VisitCountMethod UpsertOneDistrictVisitStat ", log.E(err))
return err
}
}
}
return nil
}
// NewDirectCountMethod 新增
func NewDirectCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
type districtRegister struct {
NewDirectCount int64
NewFissionCount int64
NewDeductionCount int64
}
for _, appid := range appIds {
rechangeUserMap := make(map[string]*districtRegister)
// 获取充值金额
err := registermod.RegisterUsersInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(registerInfo *registermod.RegisterMod) {
if len(registerInfo.DistrictCode) > 0 {
if registerInfo.IsDeduction {
if value, ok := rechangeUserMap[registerInfo.DistrictCode]; !ok {
rechangeUserMap[registerInfo.DistrictCode] = &districtRegister{
NewDeductionCount: 1,
}
} else {
value.NewDeductionCount += 1
}
} else {
if registerInfo.IsDirect {
if value, ok := rechangeUserMap[registerInfo.DistrictCode]; !ok {
rechangeUserMap[registerInfo.DistrictCode] = &districtRegister{
NewDirectCount: 1,
}
} else {
value.NewDirectCount += 1
}
} else {
if value, ok := rechangeUserMap[registerInfo.DistrictCode]; !ok {
rechangeUserMap[registerInfo.DistrictCode] = &districtRegister{
NewFissionCount: 1,
}
} else {
value.NewFissionCount += 1
}
}
}
}
})
if err != nil {
log.Error("NewRechargeCountMethod TotalDistricRechargeAmount ", log.E(err))
return err
}
// 插入数据库
for k, v := range rechangeUserMap {
// 更新渠道存
err := channelCodeBlocStatMod.UpsertOneUserRegisterStat(ctx, k, appid, start, v.NewDirectCount, v.NewFissionCount, v.NewDeductionCount)
if err != nil {
log.Error("VisitCountMethod UpsertOneDistrictVisitStat ", log.E(err))
return err
}
}
}
return nil
}
skd/job/blocstatdistrict/stat.go
+89
View File
@@ -0,0 +1,89 @@
package blocstatdistrict
import (
"context"
"stat/common/log"
"stat/common/statutils"
"stat/common/utils"
"stat/models/appInfoMod"
statrecordmod "stat/models/statrecord"
"time"
)
/*
* 主要统计任务:
* 1、渠道每天的：直推访问数、裂变访问数、次日留存率、3日留存率、7日留存率。
*/
func RunStat(defaultStatDate time.Time) error {
log.Info("statdistrict stat running...")
ctx := context.Background()
recordMap := defaultTimeMap(defaultStatDate)
// 1.获取统计项上次统计的时候(每天执行)
recoredResult, err := statrecordmod.LastRecordTimeMap(ctx, statrecordmod.BlocDiscStatJob) //第一次从默认记录时间开始统计
if err != nil {
log.Error("statdistrict RunStat lastRecordTime failed!", log.E(err))
return err
}
for _, record := range recoredResult {
recordMap[record.Item] = record.RecordTime
}
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("statdistrict RunStat GetAppIds failed!", log.E(err))
return err
}
statFrq := 24 * time.Hour // 统计频率
curStatTime := utils.BeginOfToday() // 本次统计时间
for item, lastStatAt := range recordMap {
_lastStatAt := lastStatAt
for curStatTime.After(_lastStatAt) {
end := _lastStatAt.Add(statFrq)
// 超过本次统计结束时间，则结束
if end.After(curStatTime) {
end = curStatTime
}
// 根据时间段统计数据
err := runStatMethod(ctx, item, _lastStatAt, end, appIds)
if err != nil {
log.Error("statdistrict statByTimeRange failed!", log.E(err), log.Any("item", item))
return err
}
_lastStatAt = end
//fmt.Println(_lastStatAt)
}
recordMap[item] = _lastStatAt
}
return nil
}
// runStatMethod 调用统计项方法
func runStatMethod(ctx context.Context, item string, start, end time.Time, appIds []int32) (err error) {
// 1.获取统计方法
statMethod, ok := statMethodMap[item]
if !ok {
return nil
}
//fmt.Println(item)
// 调用统计方法，获取数据
err = statutils.ServerMon.RunWithName(func() error {
return statMethod(ctx, start, end, appIds)
}, statutils.GetFuncName(statMethod))
if err != nil {
log.Error("statdistrict stat statByTimeRange statMethod failed!", log.Any("statItem", item), log.E(err))
return err
}
// 3.更新记录
if err := statrecordmod.UpsertItemRecord(ctx, statrecordmod.BlocDiscStatJob, item, end); err != nil {
log.Error("statdistrict stat commit UpsertItemRecord failed!", log.E(err))
return err
}
return nil
}
skd/job/domaincheck/method.go
+82
View File
@@ -0,0 +1,82 @@
package domaincheck
import (
"context"
"fmt"
"stat/common/log"
"stat/common/tf"
"stat/models/appInfoMod"
"stat/models/landingPage"
"stat/models/versionMod"
"stat/statg"
"strings"
)
func DomainCheck() error {
ctx := context.Background()
domainList, err := landingPage.FindAllLandingPage(ctx)
if err != nil {
log.Error("DomainCheck", log.E(err))
return err
}
availableCountCheck(ctx, domainList)
return nil
}
func availableCountCheck(ctx context.Context, domainList []landingPage.LandingPage) {
m := make(map[int32]int)
for _, v := range domainList {
m[v.AppId]++
}
var tips []string
for appID, count := range m {
log.Info("availableCountCheck", log.Any("appID", appID), log.Any("count", count))
if count <= 2 {
appinfo, _ := appInfoMod.GetAppInfoById(ctx, appID)
tips = append(tips, fmt.Sprintf("[%s]可用域名还剩:%d个.", appinfo.Name, count))
}
}
if len(tips) > 0 {
txt := strings.Join(tips, "\n")
statg.Bot.SendMonitorMessage(txt)
}
}
func TFCheck() error {
ctx := context.Background()
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("daily RunStat GetAppIds failed!", log.E(err))
return err
}
versionList := make([]versionMod.VersionModel, 0)
fmt.Println("TF域名检测 versionList", len(versionList))
for _, appid := range appIds {
version, err := versionMod.GetTfUrlList(ctx, appid)
if err != nil {
log.Error("TFCheck GetTfUrlList", log.E(err))
return err
}
versionList = append(versionList, version)
}
for _, version := range versionList {
for _, url := range version.Link {
fmt.Println("TF域名检测", url)
result, err := tf.CheckTFValid(url)
if err != nil {
log.Error("runCheckUrl CheckTFValid", log.Any("url", url), log.E(err))
continue
}
valid := result.Valid
desc := result.Desc
if !valid {
appinfo, _ := appInfoMod.GetAppInfoById(ctx, version.AppId)
text := fmt.Sprintf("[统计同学]:应用名称:%s\ntf域名异常\nURL:%s\nversion:%s\ndesc:%s", appinfo.Name, url, version.VersionNum, desc)
statg.Bot.SendMonitorMessage(text)
}
}
}
return nil
}
skd/job/limitstat/limitstat.go
+84
View File
@@ -0,0 +1,84 @@
package limitstat
import (
"encoding/json"
"sort"
"stat/common/log"
"stat/common/redis"
"stat/service/cacheSer"
"stat/statg"
"time"
)
var mgr *redis.Manager
func getRedisMgr() (*redis.Manager, error) {
if mgr == nil {
m, err := redis.NewManagerWithUrl(statg.Conf.Redis.Url, func(opt *redis.Options) {
opt.ReadTimeout = 60 * time.Second
})
if err != nil {
return nil, err
}
mgr = m
}
return mgr, nil
}
func IpAddressLimitStat() error {
m, err := getRedisMgr()
if err != nil {
log.Error("get redis mgr failed", log.E(err))
return err
}
var piece = 1000
todayList := make(redis.BucketList, 0)
today := time.Now().Local()
todayKey := redis.GetIpLimitSpecifiedDayKey(statg.Conf.Host, statg.Conf.Port, today)
todayHash := redis.NewHash(m.Client(), todayKey)
if err := todayHash.Scan("", int64(piece), func(name, value string) {
bucket := redis.Bucket{}
if err := json.Unmarshal([]byte(value), &bucket); err == nil {
bucket.Name = name
todayList = append(todayList, &bucket)
}
}); err != nil {
return err
}
sort.Sort(todayList)
cacheSer.IpLimitStatCache.Lock.Lock()
if len(todayList) >= 1000 {
cacheSer.IpLimitStatCache.IpLimitBucketTopToday = todayList[:1000]
} else {
cacheSer.IpLimitStatCache.IpLimitBucketTopToday = todayList[:]
}
cacheSer.IpLimitStatCache.Lock.Unlock()
yesterday := today.Add(-24 * time.Hour)
yesterdayKey := redis.GetIpLimitSpecifiedDayKey(statg.Conf.Host, statg.Conf.Port, yesterday)
history := redis.NewHash(m.Client(), redis.IpLimiterDayRecordKey)
if exists, err := history.Exists(yesterdayKey); err != nil || exists {
return err
}
yesterdayList := make(redis.BucketList, 0)
yesterdayHash := redis.NewHash(m.Client(), yesterdayKey)
if err := yesterdayHash.Scan("", int64(piece), func(name, value string) {
bucket := redis.Bucket{}
if err := json.Unmarshal([]byte(value), &bucket); err == nil {
bucket.Name = name
yesterdayList = append(yesterdayList, &bucket)
}
}); err != nil {
return err
}
sort.Sort(yesterdayList)
cacheSer.IpLimitStatCache.Lock.Lock()
if len(yesterdayList) >= 1000 {
cacheSer.IpLimitStatCache.IpLimitBucketTopToday = yesterdayList[:1000]
} else {
cacheSer.IpLimitStatCache.IpLimitBucketTopToday = yesterdayList[:]
}
cacheSer.IpLimitStatCache.Lock.Unlock()
_, err = history.Set(yesterdayKey, "1")
return err
}
skd/job/monitor/method.go
+241
View File
@@ -0,0 +1,241 @@
package monitor
import (
"context"
"fmt"
"stat/common/enum/consts"
"stat/common/log"
"stat/models/appInfoMod"
"stat/models/channelmod"
"stat/models/downloadinfo"
"stat/models/registermod"
"stat/statg"
"strings"
"time"
)
type LanpageMonitor struct {
Minute int64
LastDay int64
}
var lastIosCount int64
var lastAndroidCount int64
var lastBiliIosCount int64
var lastNhavIosCount int64
var lastHuawenIosCount int64
var lastZhihuIosCount int64
var lanpages = []string{"madougg.com"}
var channels = []string{"xza00267", "xza00182", "xza00082", "xza00139", "xza00242", "sqt00120", "xza00274", "xza00258", "ysa00021", "fabuye", channelmod.OfficialChannelName, "mdapp01", "mdapp", "modelweb", "mdapp02", "mdapp03"}
func RunMonitor() error {
msg := RunLandPageMonitor()
statg.Bot.SendDownloadMessage(msg)
channelMsg := ChannelMonitor()
statg.Bot.SendDownloadMessage(channelMsg)
return nil
}
func RunLandPageMonitor() string {
ctx := context.Background()
appid := int32(888) // 麻豆
end := time.Now()
start := end.Add(-30 * time.Minute)
lanpageMap := make(map[string]*LanpageMonitor)
downloadList, serr := downloadinfo.GetAccessCountByTime(ctx, appid, start, end)
if serr != nil {
log.Error("RunMonitor, GetAccessCountByTime", log.E(serr))
return ""
}
for _, valueInfo := range downloadList {
if len(valueInfo.Domain) > 0 {
domain := valueInfo.Domain
if strings.Contains(valueInfo.Domain, "https://") {
domain = strings.Replace(valueInfo.Domain, "https://", "", -1)
} else if strings.Contains(valueInfo.Domain, "http://") {
domain = strings.Replace(valueInfo.Domain, "http://", "", -1)
}
if value, ok := lanpageMap[domain]; ok {
value.Minute += 1
} else {
lanpageMap[domain] = &LanpageMonitor{
Minute: 1,
}
}
}
}
lastDayEnd := time.Now().AddDate(0, 0, -1)
lastDayStart := lastDayEnd.Add(-30 * time.Minute)
LastDownloadList, serr := downloadinfo.GetAccessCountByTime(ctx, appid, lastDayStart, lastDayEnd)
if serr != nil {
log.Error("RunMonitor, GetAccessCountByTime", log.E(serr))
return ""
}
for _, valueInfo := range LastDownloadList {
if len(valueInfo.Domain) > 0 {
domain := valueInfo.Domain
if strings.Contains(valueInfo.Domain, "https://") {
domain = strings.Replace(valueInfo.Domain, "https://", "", -1)
} else if strings.Contains(valueInfo.Domain, "http://") {
domain = strings.Replace(valueInfo.Domain, "http://", "", -1)
}
if value, ok := lanpageMap[domain]; ok {
value.LastDay += 1
} else {
lanpageMap[domain] = &LanpageMonitor{
LastDay: 1,
}
}
}
}
// 整理渠道数据
tgTxt := "落地页监控:\n"
// tgTxt += fmt.Sprintf("开始: %s\n结束: %s\n", start.Format("2006-01-02T15:04:05"), end.Format("2006-01-02T15:04:05"))
for _, lanpage := range lanpages {
for k, value := range lanpageMap {
if k == lanpage {
message := fmt.Sprintf("%s: 当日:%d 昨日:%d\n", k, value.Minute, value.LastDay)
tgTxt += message
}
}
}
if len(lanpageMap) > 0 {
return tgTxt
}
return ""
}
func ChannelMonitor() string {
ctx := context.Background()
appid := int32(888) // 麻豆
end := time.Now()
start := end.Add(-30 * time.Minute)
lastDayEnd := time.Now().AddDate(0, 0, -1)
lastDayStart := lastDayEnd.Add(-30 * time.Minute)
channelMap := make(map[string]*LanpageMonitor)
for _, channelinfo := range channels {
registerList, err := registermod.RegisterUsersInfoByDistrict(ctx, appid, channelinfo, start, end)
if err != nil {
log.Error("NewCountMethod RegisterUsersInfoByDistrict ", log.E(err))
return ""
}
for _, valueInfo := range registerList {
if valueInfo.IsDirect {
if value, ok := channelMap[valueInfo.DistrictCode]; ok {
value.Minute += 1
} else {
channelMap[valueInfo.DistrictCode] = &LanpageMonitor{
Minute: 1,
}
}
}
}
LastDayRegisterList, err := registermod.RegisterUsersInfoByDistrict(ctx, appid, channelinfo, lastDayStart, lastDayEnd)
if err != nil {
log.Error("NewCountMethod RegisterUsersInfoByDistrict ", log.E(err))
return ""
}
for _, valueInfo := range LastDayRegisterList {
if valueInfo.IsDirect {
if value, ok := channelMap[valueInfo.DistrictCode]; ok {
value.LastDay += 1
} else {
channelMap[valueInfo.DistrictCode] = &LanpageMonitor{
LastDay: 1,
}
}
}
}
}
// 自来量
fissionCount, err := registermod.FissionAddUserCount(ctx, appid, start, end)
if err != nil {
log.Error("NewCountMethod RegisterUsersInfoByDistrict ", log.E(err))
return ""
}
LastDayFissionCount, err := registermod.FissionAddUserCount(ctx, appid, lastDayStart, lastDayEnd)
if err != nil {
log.Error("NewCountMethod RegisterUsersInfoByDistrict ", log.E(err))
return ""
}
channelMap["裂变"] = &LanpageMonitor{
LastDay: LastDayFissionCount,
Minute: fissionCount,
}
// 整理渠道数据
tgTxt := "麻豆渠道监控:\n"
// tgTxt += fmt.Sprintf("开始: %s\n结束: %s\n", start.Format("2006-01-02T15:04:05"), end.Format("2006-01-02T15:04:05"))
for k, value := range channelMap {
if channelmod.IsOfficialChannelName(k) {
k = "自来量"
}
message := fmt.Sprintf("%s: 当日:%d 昨日:%d\n", k, value.Minute, value.LastDay)
tgTxt += message
}
if len(channelMap) > 0 {
return tgTxt
}
// 统计数据
return ""
}
func NewCountMonitor() error {
ctx := context.Background()
end := time.Now()
start := end.Add(-time.Minute * 10)
appIds := []int32{999, 694, 594, 494, 888, 444, 66666}
for _, appid := range appIds {
iosCount, err := registermod.IosAddUserCount(ctx, appid, start, end)
if err != nil {
log.Error("DirectVisitCountMethod IosAddUserCount", log.E(err))
return err
}
androidCount, err := registermod.AndroidAddUserCount(ctx, appid, start, end)
if err != nil {
log.Error("DirectVisitCountMethod IosAddUserCount", log.E(err))
return err
}
if statg.Conf.RunMod == consts.RunModRelease {
if androidCount <= 10 || iosCount <= 5 {
appinfo, _ := appInfoMod.GetAppInfoById(ctx, appid)
// 必须存在
if appinfo.AppId == appid {
text := fmt.Sprintf("[警告]:应用名称:%s\n2分钟新增异常:\n时间:%s\nandroid:%d\nios:%d\n", appinfo.Name, start, androidCount, iosCount)
err = statg.Bot.SendMonitorMessage(text)
if err != nil {
log.Error("tg Send ", log.E(err))
continue
}
}
}
}
}
return nil
}
skd/job/statdaily/method.go
+525
View File
@@ -0,0 +1,525 @@
package statdaily
import (
"context"
"errors"
"fmt"
"stat/common/log"
"stat/common/statutils"
"stat/common/utils"
"stat/models/accessmod"
"stat/models/blocDailystatmod"
"stat/models/dailystatmod"
"stat/models/downloadinfo"
"stat/models/rechargegamemod"
"stat/models/rechargemod"
"stat/models/registermod"
"stat/models/syncrecharge"
"stat/models/syncvip"
"stat/models/versionMod"
"stat/models/withdrawGameOrdermod"
"strconv"
"strings"
"time"
"go.mongodb.org/mongo-driver/bson"
)
const (
AppVisitCount string = "appVisitCount" // app访问数
Day1RetainRate string = "day1RetainRate" // 次日留存率
Day3RetainRate string = "day3RetainRate" // 3日留存率
Day7RetainRate string = "day7RetainRate" // 7日留存率
RechargeAmount string = "rechargeAmount" // 充值金额
NewDirectCount string = "newDirectCount" // 直推
VipRechargeAmount string = "vipRechargeAmount" // VIP购买数
RechargeCount string = "rechargeCount" // 充值次数
GameRecharge string = "gameRecharge" // 游戏充值
LandpageClick string = "landpageClick" // 落地页下载按钮点击次数
)
func defaultTimeMap(defaultStatDate time.Time) map[string]time.Time {
return map[string]time.Time{
AppVisitCount: defaultStatDate, // app访问人数
Day1RetainRate: defaultStatDate, // 次日留存率
Day3RetainRate: defaultStatDate, // 3日留存率
Day7RetainRate: defaultStatDate, // 7日留存率
RechargeAmount: defaultStatDate, // 直推充值金额
NewDirectCount: defaultStatDate, // 新增直推
VipRechargeAmount: defaultStatDate, // VIP购买数
RechargeCount: defaultStatDate, // 充值次数
GameRecharge: defaultStatDate, // 游戏充值
LandpageClick: defaultStatDate, // 落地页
}
}
// 获取统计方法
type StatMethod = func(ctx context.Context, start time.Time, end time.Time, appIds []int32) error
// 统计方法汇总
var statMethodMap = map[string]StatMethod{
AppVisitCount: AppVisitCountMethod, // 访问数
Day1RetainRate: Day1RetainRateMethod, // 次日留存率
Day3RetainRate: Day3RetainRateMethod, // 3日留存率
Day7RetainRate: Day7RetainRateMethod, // 7日留存率
RechargeAmount: AmountMethod, // 直推充值金额
NewDirectCount: NewCountMethod, // 注册用户
VipRechargeAmount: VipRechargeAmountMethod, // VIP购买数
RechargeCount: RechargeCountMethod, // 充值次数
GameRecharge: GameRechargeMethod, // 游戏充值
LandpageClick: LandpageClickMethod, // 落地页
}
// AppVisitCountMethod app访问数
func AppVisitCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
stat := &statutils.Stat{}
err := accessmod.VisitInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *accessmod.AccessMod) {
client := statutils.GetClientInfo(item.SysType, item.DevType,
item.DistrictCode, item.IsDirect, false, item.IsFormalUser)
stat.Add(1, client)
})
if err != nil {
log.Error("AppVisitCountMethod:VisitInfoByTimeShard", log.E(err))
return err
}
//更新数据库
doc := dailystatmod.DailyStatDoc{
TotalVisitCount: stat.CountRef(),
DirectVisitCount: stat.Total.Direct.CountRef(),
DeductionVisitCount: stat.Total.Deduction.CountRef(),
NaturalVisitCount: stat.Total.Direct.Natural.CountRef(),
FissionVisitCount: stat.Total.Fission.CountRef(),
TodayDistrictVisitCount: stat.Total.District.CountRef(),
TodayWebVisitCount: stat.Total.Web.CountRef(),
TodayNormalVisitCount: stat.Total.Normal.CountRef(),
TodayIosVisitCount: stat.Ios.CountRef(),
TodayAndroidVisitCount: stat.Android.CountRef(),
LightVisitCount: stat.Light.CountRef(),
LightDirectVisitCount: stat.Light.Total.Direct.CountRef(),
LightFissionVisitCount: stat.Light.Total.Fission.CountRef(),
LightDeductionVisitCount: stat.Light.Total.Deduction.CountRef(),
LightNaturalVisitCount: stat.Light.Total.Direct.Natural.CountRef(),
LightWebVisitCount: stat.Light.Total.Web.CountRef(),
LightDistrictVisitCount: stat.Light.Total.District.CountRef(),
LightNormalVisitCount: stat.Light.Total.Normal.CountRef(),
TodayH5VisitCount: stat.H5.CountRef(),
TodayH5PcVisitCount: stat.H5.Pc.CountRef(),
TodayH5IosVisitCount: stat.H5.Ios.CountRef(),
TodayH5AndroidVisitCount: stat.H5.Android.CountRef(),
}
err = dailystatmod.UpsertDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("H5appVisitCountMethod:UpsertDailyStat", log.E(err))
return err
}
}
return nil
}
// Day1RetainRateMethod 次日留存率
func Day1RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
rate := float64(0)
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -1))
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -1))
registerCount, err := registermod.TotalAddUserCount(ctx, appid, registerStart, registerEnd)
if err != nil {
log.Error("Day1RetainRateMethod s", log.E(err))
return err
}
VistitCount, err := accessmod.DailyUserVisitCountByRegister(ctx, appid, start, end, registerStart, registerEnd)
if err != nil {
log.Error("Day1RetainRateMethod DailyUserVisitCountByRegister", log.E(err))
return err
}
if registerCount > 0 {
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(VistitCount)/float64(registerCount))), 64)
//fmt.Println(VistitCount, registerCount, rate)
}
// 更新数据库
doc := dailystatmod.DailyStatDoc{
Day1RetainRate: &rate,
}
err = dailystatmod.UpsertDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("Day1RetainRateMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// Day3RetainRateMethod 3日留存率
func Day3RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
rate := float64(0)
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -2))
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -2))
registerCount, err := registermod.TotalAddUserCount(ctx, appid, registerStart, registerEnd)
if err != nil {
log.Error("Day1RetainRateMethod s", log.E(err))
return err
}
VistitCount, err := accessmod.DailyUserVisitCountByRegister(ctx, appid, start, end, registerStart, registerEnd)
if err != nil {
log.Error("Day1RetainRateMethod DailyUserVisitCountByRegister", log.E(err))
return err
}
if registerCount > 0 {
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(VistitCount)/float64(registerCount))), 64)
}
// 更新数据库
doc := dailystatmod.DailyStatDoc{
Day3RetainRate: &rate,
}
err = dailystatmod.UpsertDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("Day3RetainRateMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// Day7RetainRateMethod 7日留存率
func Day7RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
rate := float64(0)
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -6))
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -6))
registerCount, err := registermod.TotalAddUserCount(ctx, appid, registerStart, registerEnd)
if err != nil {
log.Error("Day1RetainRateMethod s", log.E(err))
return err
}
VistitCount, err := accessmod.DailyUserVisitCountByRegister(ctx, appid, start, end, registerStart, registerEnd)
if err != nil {
log.Error("Day1RetainRateMethod DailyUserVisitCountByRegister", log.E(err))
return err
}
if registerCount > 0 {
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(VistitCount)/float64(registerCount))), 64)
}
// 更新数据库
doc := dailystatmod.DailyStatDoc{
Day7RetainRate: &rate,
}
err = dailystatmod.UpsertDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("Day7RetainRateMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// AmountMethod 直推充值金额
func AmountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
// 获取充值金额
itemList, err := rechargemod.TotalRechargeListV1(ctx, appid, start, end)
if err != nil {
log.Error("DirectAmountMethod DirectRechargeAmount ", log.E(err))
return err
}
stat := &statutils.Stat{}
repStat := &statutils.Stat{}
newStat := &statutils.Stat{}
for _, item := range itemList {
client := statutils.GetClientInfo(item.SysType, item.DevType, item.DistrictCode, item.IsDirect, false, item.IsFormalUser)
stat.Add(item.PayMoney, client)
if item.RegisterTime.After(item.SuccessAt.AddDate(0, 0, -1)) {
newStat.Add(item.PayMoney, client)
}
if item.Repeat {
repStat.Add(item.PayMoney, client)
}
}
// 更新数据库
doc := dailystatmod.DailyStatDoc{
TotalAmount: stat.ValueRef(),
SuccessRechargeOrder: stat.CountRef(),
DirectAmount: stat.Total.Direct.ValueRef(),
DirectAmountCount: stat.Total.Direct.CountRef(),
FissionAmount: stat.Total.Fission.ValueRef(),
FissionAmountCount: stat.Total.Fission.CountRef(),
NaturalAmount: stat.Total.Direct.Natural.ValueRef(),
NaturalAmountCount: stat.Total.Direct.Natural.CountRef(),
DeductionAmount: stat.Total.Deduction.ValueRef(),
DeductionAmountCount: stat.Total.Deduction.CountRef(),
TodayWebRecharge: stat.Total.Web.ValueRef(),
TodayDistrictRecharge: stat.Total.District.ValueRef(),
TodayNormalRecharge: stat.Total.Normal.ValueRef(),
TodayAndroidRecharge: stat.Android.ValueRef(),
TodayIosRecharge: stat.Ios.ValueRef(),
LightRechargeCount: stat.Light.CountRef(),
LightDirectRechargeCount: stat.Light.Total.Direct.CountRef(),
LightFissionRechargeCount: stat.Light.Total.Fission.CountRef(),
LightDeductionRechargeCount: stat.Light.Total.Deduction.CountRef(),
LightNaturalRechargeCount: stat.Light.Total.Direct.Natural.CountRef(),
LightWebRechargeCount: stat.Light.Total.Web.CountRef(),
LightDistrictRechargeCount: stat.Light.Total.District.CountRef(),
LightNormalRechargeCount: stat.Light.Total.Normal.CountRef(),
LightRechargeAmount: stat.Light.ValueRef(),
LightDirectRechargeAmount: stat.Light.Total.Direct.ValueRef(),
LightFissionRechargeAmount: stat.Light.Total.Fission.ValueRef(),
LightDeductionRechargeAmount: stat.Light.Total.Deduction.ValueRef(),
LightNaturalRechargeAmount: stat.Light.Total.Direct.Natural.ValueRef(),
LightWebRechargeAmount: stat.Light.Total.Web.ValueRef(),
LightDistrictRechargeAmount: stat.Light.Total.District.ValueRef(),
LightNormalRechargeAmount: stat.Light.Total.Normal.ValueRef(),
TodayH5Recharge: stat.H5.ValueRef(),
TodayH5IosRecharge: stat.H5.Ios.ValueRef(),
TodayH5AndroidRecharge: stat.H5.Android.ValueRef(),
TodayH5PcRecharge: stat.H5.Pc.ValueRef(),
NewRechargeAmount: newStat.ValueRef(),
NewRechargeCount: newStat.CountRef(),
RepeatAmount: repStat.ValueRef(),
RepeatCount: repStat.CountRef(),
}
err = dailystatmod.UpsertDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("DirectAmountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
func NewCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
stat := &statutils.Stat{}
userDict := make(map[uint64]bool)
err := registermod.RegisterFormalsInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
if item.IsFormal && statutils.IsClientH5OrLight(item.SysType, item.DevType) {
if _, exists := userDict[item.UserId]; !exists {
userDict[item.UserId] = true
client := statutils.GetClientInfo(item.SysType, item.DevType,
item.DistrictCode, item.IsDirect, false, item.IsFormal)
stat.Add(1, client)
}
}
})
if err != nil {
return err
}
err = registermod.RegisterUsersInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
userDict[item.UserId] = true
client := statutils.GetClientInfo(item.SysType, item.DevType,
item.DistrictCode, item.IsDirect, false, item.IsFormal)
stat.Add(1, client)
})
if err != nil {
return err
}
doc := dailystatmod.DailyStatDoc{
NewTotalCount: stat.CountRef(),
NewDirectCount: stat.Total.Direct.CountRef(),
NewFissionCount: stat.Total.Fission.CountRef(),
NewDeductionCount: stat.Total.Deduction.CountRef(),
NewNaturalCount: stat.Total.Direct.Natural.CountRef(),
NewWebCount: stat.Total.Web.CountRef(),
NewDistrictCount: stat.Total.District.CountRef(),
NewNormalCount: stat.Total.Normal.CountRef(),
AndroidAddCount: stat.Android.CountRef(),
IosAddCount: stat.Ios.CountRef(),
LightAddCount: stat.Light.CountRef(),
LightGuestAddCount: stat.Light.Guest.CountRef(),
LightFormalAddCount: stat.Light.Formal.CountRef(),
LightDirectCount: stat.Light.Total.Direct.CountRef(),
LightFissionCount: stat.Light.Total.Fission.CountRef(),
LightDeductionCount: stat.Light.Total.Deduction.CountRef(),
LightNaturalCount: stat.Light.Total.Direct.Natural.CountRef(),
LightWebCount: stat.Light.Total.Web.CountRef(),
LightDistrictCount: stat.Light.Total.District.CountRef(),
LightNormalCount: stat.Light.Total.Normal.CountRef(),
H5AddCount: stat.H5.CountRef(),
H5PcAddCount: stat.H5.Pc.CountRef(),
H5IosAddCount: stat.H5.Ios.CountRef(),
H5AndroidAddCount: stat.H5.Android.CountRef(),
H5FormalCount: stat.H5.Formal.CountRef(),
}
err = dailystatmod.UpsertDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("NewDeductionCountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// VipRechargeAmountMethod VIP购买次数
func VipRechargeAmountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
money := int64(0)
amount := int64(0)
NewVipPayMoney := int64(0)
RepeatVipMoney := int64(0)
filter := bson.M{syncvip.FN.SyncVip.AppId: appid, syncvip.FN.SyncVip.SuccessAt: bson.M{"$gt": start, "$lt": end}}
viporderList, serr := syncvip.FindManySyncVip(ctx, filter)
if serr != nil {
return errors.New(serr.Error())
}
for _, viporder := range viporderList {
money += viporder.Money
amount += viporder.Amount
if viporder.RegisterTime.After(viporder.SuccessAt.AddDate(0, 0, -1)) {
NewVipPayMoney += viporder.Money
}
if viporder.Repeat {
RepeatVipMoney += viporder.Money
}
}
// 更新数据库
doc := dailystatmod.DailyStatDoc{
VipPayByMoney: &money,
VipPayByAmount: &amount,
NewVipPayMoney: &NewVipPayMoney,
RepeatVipMoney: &RepeatVipMoney,
}
err := dailystatmod.UpsertDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("NewDeductionCountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// RechargeCountMethod 充值次数
func RechargeCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
filter := bson.M{syncvip.FN.SyncVip.AppId: appid, syncrecharge.FN.SyncRechargeOrder.CreatedAt: bson.M{"$gt": start, "$lt": end}}
total, serr := syncrecharge.CountSyncRechargeOrders(ctx, filter)
if serr != nil {
return errors.New(serr.Error())
}
doc := dailystatmod.DailyStatDoc{
TotalRechargeOrder: &total,
}
err := dailystatmod.UpsertDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("NewDeductionCountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// GameRechargeMethod 游戏充值
func GameRechargeMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
rechageAmount := int64(0)
withdrawAmount := int64(0)
withdrawList, serr := withdrawGameOrdermod.GetGameWithdrawOrder(ctx, appid, start, end)
if serr != nil {
return errors.New(serr.Error())
}
for _, withdrawInfo := range withdrawList {
withdrawAmount += withdrawInfo.PayMoney
}
filter := bson.M{"appId": appid, "successAt": bson.M{"$gt": start, "$lte": end}}
rechageList, serr := rechargegamemod.FindManyRechargeInfo(ctx, filter)
if serr != nil {
return errors.New(serr.Error())
}
for _, rechageInfo := range rechageList {
rechageAmount += rechageInfo.PayMoney
}
// 更新数据库
doc := dailystatmod.DailyStatDoc{
GameRechageAmount: &rechageAmount,
GameWithdrawAmount: &withdrawAmount,
}
err := dailystatmod.UpsertDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("NewDeductionCountMethod UpsertDailyStat ", log.E(err))
return err
}
// 更新数据库
blocDoc := blocDailystatmod.BlocDailyStatDoc{
GameRechageAmount: &rechageAmount,
GameWithdrawAmount: &withdrawAmount,
}
err = blocDailystatmod.UpsertBlocDailyStat(ctx, start, appid, blocDoc)
if err != nil {
log.Error("DeductionAmountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// LandpageClickMethod 落地页访问次数
func LandpageClickMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
downloadList, err := downloadinfo.GetAccessCountByTime(ctx, appid, start, end)
if err != nil {
log.Error("RunMonitor, GetAccessCountByTime", log.E(err))
return err
}
androidClick := int64(0)
appstoreClick := int64(0)
iosAtherClick := int64(0)
lightClick := int64(0)
for _, downloadInfo := range downloadList {
if downloadInfo.SysType == "android" {
androidClick += 1
} else if strings.Contains(downloadInfo.SysType, "ios") || strings.Contains(downloadInfo.SysType, "light") {
if downloadInfo.PackageType == versionMod.Shop {
appstoreClick += 1
} else if downloadInfo.PackageType == versionMod.Enterprise {
iosAtherClick += 1
} else if downloadInfo.PackageType == versionMod.Light {
lightClick += 1
}
}
}
// 更新数据库
doc := dailystatmod.DailyStatDoc{
LandpageAndroidClick: &androidClick,
LandpageIosAppstoreClick: &appstoreClick,
LandpageIosAthorClick: &iosAtherClick,
LandpageIosH5Click: &lightClick,
}
err = dailystatmod.UpsertDailyStat(ctx, start, appid, doc)
if err != nil {
log.Error("NewDeductionCountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
skd/job/statdaily/stat.go
+119
View File
@@ -0,0 +1,119 @@
package statdaily
import (
"context"
"stat/common/log"
"stat/common/statutils"
"stat/common/utils"
"stat/models/appInfoMod"
statrecordmod "stat/models/statrecord"
"time"
)
/*
* 主要统计任务:
* 1、统计每天的app的直推访问数、裂变访问数、次日留存率、3日留存率、7日留存率。（表格）
* 2、按5分钟一个点，记录实时数据 app（新增、访问数....）。（曲线图）
*/
// RunDayStat 任务1:统计前一天日活（每天1点开始统计）
func RunStat(defaultStatDate time.Time) error {
log.Info("daily stat running...")
ctx := context.Background()
recordMap := defaultTimeMap(defaultStatDate)
// 1.获取统计项上次统计的时候(每天执行)
recoredResult, err := statrecordmod.LastRecordTimeMap(ctx, statrecordmod.DailyStatDayJob) //第一次从默认记录时间开始统计
if err != nil {
log.Error("Daily RunStat lastRecordTime failed!", log.E(err))
return err
}
for _, record := range recoredResult {
recordMap[record.Item] = record.RecordTime
}
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("daily RunStat GetAppIds failed!", log.E(err))
return err
}
statFrq := 24 * time.Hour // 统计频率
curStatTime := utils.BeginOfToday() // 本次统计时间
for item, lastStatAt := range recordMap {
_lastStatAt := lastStatAt
for curStatTime.After(_lastStatAt) {
end := _lastStatAt.Add(statFrq)
// 超过本次统计结束时间，则结束
if end.After(curStatTime) {
end = curStatTime
}
// 根据时间段统计数据
err := runStatMethod(ctx, item, _lastStatAt, end, appIds)
if err != nil {
log.Error("daily statByTimeRange failed!", log.E(err), log.Any("item", item))
return err
}
_lastStatAt = end
}
recordMap[item] = _lastStatAt
}
return nil
}
// 实时统计每天的情况
func RunStatToday() error {
ctx := context.Background()
now := time.Now().Add(time.Minute * -5)
start := utils.BeginWithTime(now)
end := utils.EndWithTime(now)
log.Info("daily today stat running...")
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("daily today RunStat GetAppIds failed!", log.E(err))
return err
}
var lastErr error
for item, statMethod := range statMethodMap {
// 调用统计方法，获取数据
err = statutils.ServerMon.RunWithName(func() error {
return statMethod(ctx, start, end, appIds)
}, statutils.GetFuncName(statMethod))
if err != nil {
lastErr = err
log.Error("Daily today stat statByTimeRange statMethod failed!", log.Any("statItem", item), log.E(err))
}
}
return lastErr
}
// runStatMethod 调用统计项方法
func runStatMethod(ctx context.Context, item string, start, end time.Time, appIds []int32) (err error) {
// 1.获取统计方法
statMethod, ok := statMethodMap[item]
if !ok {
return nil
}
// 调用统计方法，获取数据
err = statutils.ServerMon.RunWithName(func() error {
return statMethod(ctx, start, end, appIds)
}, statutils.GetFuncName(statMethod))
if err != nil {
log.Error("Daily stat statByTimeRange statMethod failed!", log.Any("statItem", item), log.E(err))
return err
}
// 3.更新记录
if err := statrecordmod.UpsertItemRecord(ctx, statrecordmod.DailyStatDayJob, item, end); err != nil {
log.Error("Daily stat commit UpsertItemRecord failed!", log.E(err))
return err
}
return nil
}
skd/job/statdaily/stat2.go
+39
View File
@@ -0,0 +1,39 @@
package statdaily
import (
"context"
"fmt"
"stat/models/dailystatmod"
"time"
)
// 查询日活
func RunStatIos() {
appId := int32(999)
start := time.Now().AddDate(0, -5, 0)
end := time.Now()
statList, _ := dailystatmod.GetDailyStatTotalList(context.Background(), &appId, &start, &end)
for _, statInfo := range statList {
//占比
iosrechage := statInfo.TotalAmount - statInfo.TodayAndroidRecharge
iosStore := statInfo.TodayH5Recharge
iosrechageRate := float64(iosrechage) / float64(statInfo.TotalAmount)
iosStoreRate := float64(iosStore) / float64(statInfo.TotalAmount)
iosNew := statInfo.NewTotalCount - statInfo.AndroidAddCount
iosNewRate := float64(iosNew) / float64(statInfo.NewTotalCount)
iosStoreNew := statInfo.H5AddCount
iosStoreNewRate := float64(iosStoreNew) / float64(statInfo.NewTotalCount)
iosVisit := statInfo.TotalVisitCount - statInfo.TodayAndroidVisitCount
iosVisitRate := float64(iosVisit) / float64(statInfo.TotalVisitCount)
iosStoreVisit := statInfo.TodayH5VisitCount
iosStoreVisitRate := float64(iosStoreNew) / float64(statInfo.TotalVisitCount)
text := fmt.Sprintf("日期:%s\n", statInfo.SumDate.Format("2006-01-02"))
text += fmt.Sprintf(" 总充值:%d 总新增:%d\n", statInfo.TotalAmount/100, statInfo.NewTotalCount)
text += fmt.Sprintf(" Ios充值:%d(占比:%.2f) 商店包充值:%d(占比:%.2f)\n", iosrechage/100, iosrechageRate*100, iosStore/100, iosStoreRate*100)
text += fmt.Sprintf(" Ios新增:%d(占比:%.2f) 商店包新增:%d(占比:%.2f)\n", iosNew, iosNewRate*100, iosStoreNew, iosStoreNewRate*100)
text += fmt.Sprintf(" Ios日活:%d(占比:%.2f) 商店包日活:%d(占比:%.2f)\n", iosVisit, iosVisitRate*100, iosStoreVisit, iosStoreVisitRate*100)
fmt.Println(text)
}
}
skd/job/statdailyhour/method.go
+395
View File
@@ -0,0 +1,395 @@
package statdailyhour
import (
"context"
"errors"
"stat/common/log"
"stat/common/statutils"
"stat/common/utils"
"stat/models/accessmod"
"stat/models/dailyhourstatmod"
"stat/models/downloadinfo"
"stat/models/rechargegamemod"
"stat/models/rechargemod"
"stat/models/registermod"
"stat/models/syncrecharge"
"stat/models/syncvip"
"stat/models/versionMod"
"stat/models/withdrawGameOrdermod"
"strings"
"time"
"go.mongodb.org/mongo-driver/bson"
)
const (
AppVisitCount string = "appVisitCount" // app访问数
RechargeAmount string = "rechargeAmount" // 直推充值金额
NewCount string = "newDirectCount" // 新增直推
VipRechargeAmount string = "vipRechargeAmount" // VIP购买数
RechargeOrderCount string = "RechargeOrder" // VIP购买数
GameRecharge string = "GameRecharget" // 游戏充值
LandpageClick string = "landpageClick" // 游戏充值
)
func defaultTimeMap(defaultStatDate time.Time) map[string]time.Time {
return map[string]time.Time{
AppVisitCount: defaultStatDate, // App访问人数
RechargeAmount: defaultStatDate, // 直推充值金额
NewCount: defaultStatDate, // 新增直推
VipRechargeAmount: defaultStatDate, // VIP购买数
RechargeOrderCount: defaultStatDate, // 发起充值数
GameRecharge: defaultStatDate, // 游戏充值
LandpageClick: defaultStatDate, // 落地页
}
}
// 获取统计方法
type StatMethod = func(ctx context.Context, start time.Time, end time.Time, appIds []int32) error
// 统计方法汇总
var statMethodMap = map[string]StatMethod{
AppVisitCount: AppVisitCountMethod, // App访问人数
RechargeAmount: RechargeAmountMethod, // 直推充值金额
NewCount: NewCountMethod, // 新增数
VipRechargeAmount: VipRechargeAmountMethod, // VIP购买数
RechargeOrderCount: RechargeCountMethod, // 发起充值数
GameRecharge: GameRechargeMethod, // 游戏充值
LandpageClick: LandpageClickMethod, // 落地页
}
// AppVisitCountMethod app访问数
func AppVisitCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
stat := &statutils.Stat{}
err := accessmod.VisitInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *accessmod.AccessMod) {
client := statutils.GetClientInfo(item.SysType, item.DevType, item.DistrictCode, item.IsDirect, false, item.IsFormalUser)
stat.Add(1, client)
})
if err != nil {
log.Error("AppVisitCountMethod ", log.E(err))
return err
}
//更新数据库
doc := dailyhourstatmod.DailyHourStatDoc{
TotalVisitCount: stat.CountRef(),
DirectVisitCount: stat.Total.Direct.CountRef(),
NaturalVisitCount: stat.Total.Direct.Natural.CountRef(),
FissionVisitCount: stat.Total.Fission.CountRef(),
DeductionVisitCount: stat.Total.Deduction.CountRef(),
IosVisitCount: stat.Ios.CountRef(),
AndroidVisitCount: stat.Android.CountRef(),
LightVisitCount: stat.Light.CountRef(),
LightVisitDirectCount: stat.Light.Total.Direct.CountRef(),
LightVisitFissionCount: stat.Light.Total.Fission.CountRef(),
LightVisitDeductionCount: stat.Light.Total.Deduction.CountRef(),
LightVisitNaturalCount: stat.Light.Total.Direct.Natural.CountRef(),
LightVisitWebCount: stat.Light.Total.Web.CountRef(),
LightVisitNormalCount: stat.Light.Total.Normal.CountRef(),
LightVisitDistrictCount: stat.Light.Total.District.CountRef(),
H5VisitCount: stat.H5.CountRef(),
H5PcVisitCount: stat.H5.Pc.CountRef(),
H5IosVisitCount: stat.H5.Ios.CountRef(),
H5AndroidVisitCount: stat.H5.Android.CountRef(),
WebVisitCount: stat.Total.Web.CountRef(),
NormalVisitCount: stat.Total.Normal.CountRef(),
DistrictVisitCount: stat.Total.District.CountRef(),
}
err = dailyhourstatmod.UpsertDailyHourStat(ctx, end, appid, doc)
if err != nil {
log.Error("DirectAmountMethod UpsertDailyHourStat ", log.E(err))
return err
}
}
return nil
}
// RechargeAmountMethod 充值金额
func RechargeAmountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
// 获取充值金额
itemList, err := rechargemod.TotalRechargeListV1(ctx, appid, start, end)
if err != nil {
log.Error("DirectAmountMethod DirectRechargeAmount ", log.E(err))
return err
}
stat := &statutils.Stat{}
repStat := &statutils.Stat{}
newStat := &statutils.Stat{}
for _, item := range itemList {
client := statutils.GetClientInfo(item.SysType, item.DevType, item.DistrictCode, item.IsDirect, false, item.IsFormalUser)
stat.Add(item.Money, client)
if item.RegisterTime.After(item.SuccessAt.AddDate(0, 0, -1)) {
newStat.Add(item.Money, client)
}
if item.Repeat {
repStat.Add(item.Money, client)
}
}
// 更新数据库
doc := dailyhourstatmod.DailyHourStatDoc{
TotalAmount: stat.ValueRef(),
SuccessRechargeOrder: stat.CountRef(),
DirectAmount: stat.Total.Direct.ValueRef(),
DirectAmountCount: stat.Total.Direct.CountRef(),
FissionAmount: stat.Total.Fission.ValueRef(),
FissionAmountCount: stat.Total.Fission.CountRef(),
NaturalAmount: stat.Total.Direct.Natural.ValueRef(),
NaturalAmountCount: stat.Total.Direct.Natural.CountRef(),
DeductionAmount: stat.Total.Deduction.ValueRef(),
DeductionAmountCount: stat.Total.Deduction.CountRef(),
AndroidAmount: stat.Android.ValueRef(),
AndroidAmountCount: stat.Android.CountRef(),
IosAmount: stat.Ios.ValueRef(),
IosAmountCount: stat.Ios.CountRef(),
LightAmount: stat.Light.ValueRef(),
LightDirectAmount: stat.Light.Total.Direct.ValueRef(),
LightFissionAmount: stat.Light.Total.Fission.ValueRef(),
LightDeductionAmount: stat.Light.Total.Deduction.ValueRef(),
LightNaturalAmount: stat.Light.Total.Direct.Natural.ValueRef(),
LightWebAmount: stat.Light.Total.Web.ValueRef(),
LightNormalAmount: stat.Light.Total.Normal.ValueRef(),
LightDistrictAmount: stat.Light.Total.District.ValueRef(),
LightAmountCount: stat.Light.CountRef(),
LightDirectAmountCount: stat.Light.Total.Direct.CountRef(),
LightFissionAmountCount: stat.Light.Total.Fission.CountRef(),
LightDeductionAmountCount: stat.Light.Total.Deduction.CountRef(),
LightNaturalAmountCount: stat.Light.Total.Direct.Natural.CountRef(),
LightWebAmountCount: stat.Light.Total.Web.CountRef(),
LightNormalAmountCount: stat.Light.Total.Normal.CountRef(),
LightDistrictAmountCount: stat.Light.Total.District.CountRef(),
H5Amount: stat.H5.ValueRef(),
H5AmountCount: stat.H5.CountRef(),
H5FormalAmount: stat.H5.Formal.ValueRef(),
H5FormalAmountCount: stat.H5.Formal.CountRef(),
H5PcAmount: stat.H5.Pc.ValueRef(),
H5PcAmountCount: stat.H5.Pc.CountRef(),
H5IosAmount: stat.H5.Ios.ValueRef(),
H5IosAmountCount: stat.H5.Ios.CountRef(),
H5AndroidAmount: stat.Android.ValueRef(),
H5AndroidAmountCount: stat.Android.CountRef(),
WebAmount: stat.Total.Web.ValueRef(),
WebAmountCount: stat.Total.Web.CountRef(),
NormalAmount: stat.Total.Normal.ValueRef(),
NormalAmountCount: stat.Total.Normal.CountRef(),
DistrictAmount: stat.Total.District.ValueRef(),
DistrictAmountCount: stat.Total.District.CountRef(),
RepeatAmount: repStat.ValueRef(),
RepeatAmountCount: repStat.CountRef(),
NewRechargeAmount: newStat.ValueRef(),
NewRechargeCount: newStat.CountRef(),
}
err = dailyhourstatmod.UpsertDailyHourStat(ctx, end, appid, doc)
if err != nil {
log.Error("DirectAmountMethod UpsertDailyHourStat ", log.E(err))
return err
}
}
return nil
}
// NewCountMethod 新增直推
func NewCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
stat := &statutils.Stat{}
userDict := make(map[uint64]bool)
err := registermod.RegisterFormalsInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
if item.IsFormal && statutils.IsClientH5OrLight(item.SysType, item.DevType) {
if _, exists := userDict[item.UserId]; !exists {
userDict[item.UserId] = true
client := statutils.GetClientInfo(item.SysType, item.DevType,
item.DistrictCode, item.IsDirect, false, item.IsFormal)
stat.Add(1, client)
}
}
})
if err != nil {
return err
}
err = registermod.RegisterUsersInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
userDict[item.UserId] = true
client := statutils.GetClientInfo(item.SysType, item.DevType,
item.DistrictCode, item.IsDirect, false, item.IsFormal)
stat.Add(1, client)
})
if err != nil {
return err
}
hour := end.Local().Hour()
// 更新数据库
doc := dailyhourstatmod.DailyHourStatDoc{
SumHour: &hour,
NewTotalCount: stat.CountRef(),
NewDirectCount: stat.Total.Direct.CountRef(),
NewFissionCount: stat.Total.Fission.CountRef(),
NewDeductionCount: stat.Total.Deduction.CountRef(),
NewNaturalCount: stat.Total.Direct.Natural.CountRef(),
NewAndroidCount: stat.Android.Total.CountRef(),
NewIosCount: stat.Ios.Total.CountRef(),
NewLightCount: stat.Light.CountRef(),
NewLightGuestCount: stat.Light.Guest.CountRef(),
NewLightFormalCount: stat.Light.Formal.CountRef(),
NewLightDirectCount: stat.Light.Total.Direct.CountRef(),
NewLightFissionCount: stat.Light.Total.Fission.CountRef(),
NewLightDeductionCount: stat.Light.Total.Deduction.CountRef(),
NewLightNaturalCount: stat.Light.Total.Direct.Natural.CountRef(),
NewLightWebCount: stat.Light.Total.Web.CountRef(),
NewLightNormalCount: stat.Light.Total.Normal.CountRef(),
NewLightDistrictCount: stat.Light.Total.District.CountRef(),
NewH5Count: stat.H5.CountRef(),
NewH5PcCount: stat.H5.Pc.CountRef(),
NewH5IosCount: stat.H5.Ios.CountRef(),
NewH5AndroidCount: stat.H5.Android.CountRef(),
NewWebCount: stat.Total.Web.CountRef(),
NewNormalCount: stat.Total.Normal.CountRef(),
NewDistrictCount: stat.Total.District.CountRef(),
}
err = dailyhourstatmod.UpsertDailyHourStat(ctx, end, appid, doc)
if err != nil {
log.Error("NewDirectCountMethod UpsertDailyHourStat ", log.E(err))
return err
}
}
return nil
}
// VipRechargeAmountMethod VIP购买次数
func VipRechargeAmountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
money := int64(0)
amount := int64(0)
repeat := int64(0)
filter := bson.M{syncvip.FN.SyncVip.AppId: appid, syncvip.FN.SyncVip.SuccessAt: bson.M{"$gt": start, "$lt": end}}
viporderList, serr := syncvip.FindManySyncVip(ctx, filter)
if serr != nil {
return errors.New(serr.Error())
}
for _, viporder := range viporderList {
money += viporder.Money
amount += viporder.Amount
if viporder.Repeat {
repeat += viporder.Money
}
}
// 更新数据库
doc := dailyhourstatmod.DailyHourStatDoc{
VipPayByMoney: &money,
VipPayByAmount: &amount,
VipRepeatAmount: &repeat,
}
err := dailyhourstatmod.UpsertDailyHourStat(ctx, end, appid, doc)
if err != nil {
log.Error("NewDeductionCountMethod UpsertDailyHourStat ", log.E(err))
return err
}
}
return nil
}
// RechargeCountMethod 充值次数
func RechargeCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
filter := bson.M{syncvip.FN.SyncVip.AppId: appid, syncrecharge.FN.SyncRechargeOrder.CreatedAt: bson.M{"$gt": start, "$lt": end}}
total, serr := syncrecharge.CountSyncRechargeOrders(ctx, filter)
if serr != nil {
return errors.New(serr.Error())
}
// 更新数据库
doc := dailyhourstatmod.DailyHourStatDoc{
TotalRechargeOrder: &total,
}
err := dailyhourstatmod.UpsertDailyHourStat(ctx, end, appid, doc)
if err != nil {
log.Error("NewDeductionCountMethod UpsertDailyHourStat ", log.E(err))
return err
}
}
return nil
}
// GameRechargeMethod 游戏充值
func GameRechargeMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
rechageAmount := int64(0)
withdrawAmount := int64(0)
withdrawList, serr := withdrawGameOrdermod.GetGameWithdrawOrder(ctx, appid, start, end)
if serr != nil {
return errors.New(serr.Error())
}
for _, withdrawInfo := range withdrawList {
withdrawAmount += withdrawInfo.Money
}
filter := bson.M{"appId": appid, "successAt": bson.M{"$gt": start, "$lt": end}}
rechageList, serr := rechargegamemod.FindManyRechargeInfo(ctx, filter)
if serr != nil {
return errors.New(serr.Error())
}
for _, rechageInfo := range rechageList {
rechageAmount += rechageInfo.Money
}
// 更新数据库
doc := dailyhourstatmod.DailyHourStatDoc{
GameRechageAmount: &rechageAmount,
GameWithdrawAmount: &withdrawAmount,
}
err := dailyhourstatmod.UpsertDailyHourStat(ctx, end, appid, doc)
if err != nil {
log.Error("NewDeductionCountMethod UpsertDailyHourStat ", log.E(err))
return err
}
}
return nil
}
// LandpageClickMethod 落地页访问次数
func LandpageClickMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
downloadList, err := downloadinfo.GetAccessCountByTime(ctx, appid, start, end)
if err != nil {
log.Error("RunMonitor, GetAccessCountByTime", log.E(err))
return err
}
androidClick := int64(0)
appstoreClick := int64(0)
iosAtherClick := int64(0)
h5Click := int64(0)
for _, downloadInfo := range downloadList {
if downloadInfo.SysType == "android" {
androidClick += 1
} else if strings.Contains(downloadInfo.SysType, "ios") || strings.Contains(downloadInfo.SysType, "light") {
if downloadInfo.PackageType == versionMod.Shop {
appstoreClick += 1
} else if downloadInfo.PackageType == versionMod.Enterprise {
iosAtherClick += 1
} else if downloadInfo.PackageType == versionMod.Light {
h5Click += 1
}
}
}
// 更新数据库
doc := dailyhourstatmod.DailyHourStatDoc{
LandpageAndroidClick: &androidClick,
LandpageIosAppstoreClick: &appstoreClick,
LandpageIosAthorClick: &iosAtherClick,
LandpageIosH5Click: &h5Click,
}
err = dailyhourstatmod.UpsertDailyHourStat(ctx, end, appid, doc)
if err != nil {
log.Error("NewDeductionCountMethod UpsertDailyHourStat ", log.E(err))
return err
}
}
return nil
}
skd/job/statdailyhour/stat.go
+107
View File
@@ -0,0 +1,107 @@
package statdailyhour
import (
"context"
"stat/common/log"
"stat/common/statutils"
"stat/common/utils"
"stat/models/appInfoMod"
statrecordmod "stat/models/statrecord"
"time"
)
func RunStat(defaultStatDate time.Time) error {
log.Info("daily stat running...")
ctx := context.Background()
recordMap := defaultTimeMap(defaultStatDate)
// 1.获取统计项上次统计的时候(每天执行)
recoredResult, err := statrecordmod.LastRecordTimeMap(ctx, statrecordmod.DailyHourStatDayJob) //第一次从默认记录时间开始统计
if err != nil {
log.Error("Daily RunStat lastRecordTime failed!", log.E(err))
return err
}
for _, record := range recoredResult {
recordMap[record.Item] = record.RecordTime
}
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("daily RunStat GetAppIds failed!", log.E(err))
return err
}
now := time.Now().Add(time.Minute * -5)
statFrq := 1 * time.Hour // 统计频率时间
curStatTime := utils.RecentMinute(now, 60) // 按60分钟对齐
for item, lastStatAt := range recordMap {
_lastStatAt := lastStatAt
for curStatTime.After(_lastStatAt) {
end := _lastStatAt.Add(statFrq)
// 超过本次统计结束时间，则结束
if end.After(curStatTime) {
end = curStatTime
}
// 根据时间段统计数据
err := runStatMethod(ctx, item, _lastStatAt, end, appIds)
if err != nil {
log.Error("daily statByTimeRange failed!", log.E(err), log.Any("item", item))
return err
}
_lastStatAt = end
}
recordMap[item] = _lastStatAt
}
return nil
}
// 实时统计每天的情况
func RunStatToday() {
ctx := context.Background()
start := utils.BeginOfToday()
end := utils.EndOfToday()
log.Info("daily today stat running...")
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("daily today RunStat GetAppIds failed!", log.E(err))
return
}
for item, statMethod := range statMethodMap {
// 调用统计方法，获取数据
err := statMethod(ctx, start, end, appIds)
if err != nil {
log.Error("Daily today stat statByTimeRange statMethod failed!", log.Any("statItem", item), log.E(err))
}
}
}
// runStatMethod 调用统计项方法
func runStatMethod(ctx context.Context, item string, start, end time.Time, appIds []int32) (err error) {
// 1.获取统计方法
statMethod, ok := statMethodMap[item]
if !ok {
return nil
}
// 调用统计方法，获取数据
err = statutils.ServerMon.RunWithName(func() error {
return statMethod(ctx, start, end, appIds)
}, statutils.GetFuncName(statMethod))
if err != nil {
log.Error("Daily stat statByTimeRange statMethod failed!", log.Any("statItem", item), log.E(err))
return err
}
// 3.更新记录
if err := statrecordmod.UpsertItemRecord(ctx, statrecordmod.DailyHourStatDayJob, item, end); err != nil {
log.Error("Daily stat commit UpsertItemRecord failed!", log.E(err))
return err
}
return nil
}
skd/job/statdistrict/method.go
+231
View File
@@ -0,0 +1,231 @@
package statdistrict
import (
"context"
"fmt"
"stat/common/log"
"stat/common/utils"
"stat/models/accessmod"
"stat/models/channelmod"
"stat/models/registermod"
"strconv"
"time"
"go.mongodb.org/mongo-driver/bson"
)
const (
Day1RetainRate string = "day1RetainRate" // 次日留存率
Day3RetainRate string = "day3RetainRate" // 3日留存率
Day7RetainRate string = "day7RetainRate" // 7日留存率
)
func defaultTimeMap(defaultStatDate time.Time) map[string]time.Time {
return map[string]time.Time{
Day1RetainRate: defaultStatDate, // 次日留存率
Day3RetainRate: defaultStatDate, // 3日留存率
Day7RetainRate: defaultStatDate, // 7日留存率
}
}
// 获取统计方法
type StatMethod = func(ctx context.Context, start time.Time, end time.Time, appIds []int32) error
// 统计方法汇总
var statMethodMap = map[string]StatMethod{
Day1RetainRate: Day1RetainRateMethod, // 次日留存率
Day3RetainRate: Day3RetainRateMethod, // 3日留存率
Day7RetainRate: Day7RetainRateMethod, // 7日留存率
}
// Day1RetainRateMethod 次日留存率
func Day1RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -1))
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -1))
for _, appid := range appIds {
districtMap := make(map[string]*registermod.DayRegisterAndVisit)
// 获取注册用户
err := registermod.RegisterUsersInfoByTimeShard(ctx, appid, registerStart, registerEnd, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
if value, ok := districtMap[item.DistrictCode]; !ok {
districtMap[item.DistrictCode] = &registermod.DayRegisterAndVisit{
RegisterCount: 1,
}
} else {
value.RegisterCount += 1
}
})
if err != nil {
log.Error("Day1RetainRateMethod RegisterUsersInfoByTimeShard", log.E(err))
return err
}
// 获取访问用户
err = accessmod.DailyUserVisitCountByInfoShard(ctx, appid, start, end, utils.GlobalShardHour, registerStart, registerEnd, func(item *accessmod.AccessMod) {
if value, ok := districtMap[item.DistrictCode]; !ok {
districtMap[item.DistrictCode] = &registermod.DayRegisterAndVisit{
TodayVisitCount: 1,
}
} else {
value.TodayVisitCount += 1
}
})
if err != nil {
log.Error("Day1RetainRateMethod DailyUserVisitCountByInfoShard", log.E(err))
return err
}
// 插入数据库
for districtCode, districtValue := range districtMap {
rate := float64(0)
if districtValue != nil && districtValue.RegisterCount > 0 {
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(districtValue.TodayVisitCount)/float64(districtValue.RegisterCount))), 64)
}
// 更新渠道留存
statFilter := bson.M{
channelmod.FN.ChannelStat.Channel: districtCode,
channelmod.FN.ChannelStat.SumDate: start,
channelmod.FN.ChannelStat.AppId: appid,
}
setDoc := bson.M{
"$set": bson.M{
channelmod.FN.ChannelStat.Day1RetainRate: rate,
},
}
serr := channelmod.UpdateOneChannelStat(ctx, statFilter, setDoc)
if serr != nil {
log.Error("Day1RetainRateMethod UpdateOneChannelStat ", log.E(err))
return err
}
}
}
return nil
}
// Day3RetainRateMethod 3日留存率
func Day3RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -2))
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -2))
for _, appid := range appIds {
districtMap := make(map[string]*registermod.DayRegisterAndVisit)
// 获取注册用户
err := registermod.RegisterUsersInfoByTimeShard(ctx, appid, registerStart, registerEnd, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
if value, ok := districtMap[item.DistrictCode]; !ok {
districtMap[item.DistrictCode] = &registermod.DayRegisterAndVisit{
RegisterCount: 1,
}
} else {
value.RegisterCount += 1
}
})
if err != nil {
log.Error("Day3RetainRateMethod RegisterUsersInfoByTimeShard", log.E(err))
return err
}
// 获取访问用户
err = accessmod.DailyUserVisitCountByInfoShard(ctx, appid, start, end, utils.GlobalShardHour, registerStart, registerEnd, func(item *accessmod.AccessMod) {
if value, ok := districtMap[item.DistrictCode]; !ok {
districtMap[item.DistrictCode] = &registermod.DayRegisterAndVisit{
TodayVisitCount: 1,
}
} else {
value.TodayVisitCount += 1
}
})
if err != nil {
log.Error("Day3RetainRateMethod DailyUserVisitCountByInfoShard", log.E(err))
return err
}
// 插入数据库
for districtCode, districtValue := range districtMap {
rate := float64(0)
if districtValue != nil && districtValue.RegisterCount > 0 {
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(districtValue.TodayVisitCount)/float64(districtValue.RegisterCount))), 64)
}
// 更新渠道留存
statFilter := bson.M{
channelmod.FN.ChannelStat.Channel: districtCode,
channelmod.FN.ChannelStat.SumDate: start,
channelmod.FN.ChannelStat.AppId: appid,
}
setDoc := bson.M{
"$set": bson.M{
channelmod.FN.ChannelStat.Day3RetainRate: rate,
},
}
serr := channelmod.UpdateOneChannelStat(ctx, statFilter, setDoc)
if serr != nil {
log.Error("Day1RetainRateMethod UpdateOneChannelStat", log.E(err))
return err
}
}
}
return nil
}
// Day7RetainRateMethod 7日留存率
func Day7RetainRateMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
registerStart := utils.BeginWithTime(start.AddDate(0, 0, -6))
registerEnd := utils.EndWithTime(start.AddDate(0, 0, -6))
for _, appid := range appIds {
districtMap := make(map[string]*registermod.DayRegisterAndVisit)
// 获取注册用户
err := registermod.RegisterUsersInfoByTimeShard(ctx, appid, registerStart, registerEnd, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
if value, ok := districtMap[item.DistrictCode]; !ok {
districtMap[item.DistrictCode] = &registermod.DayRegisterAndVisit{
RegisterCount: 1,
}
} else {
value.RegisterCount += 1
}
})
if err != nil {
log.Error("VisitCountMethod GetAccessUserByTime", log.E(err))
return err
}
// 获取访问用户
err = accessmod.DailyUserVisitCountByInfoShard(ctx, appid, start, end, utils.GlobalShardHour, registerStart, registerEnd, func(item *accessmod.AccessMod) {
if value, ok := districtMap[item.DistrictCode]; !ok {
districtMap[item.DistrictCode] = &registermod.DayRegisterAndVisit{
TodayVisitCount: 1,
}
} else {
value.TodayVisitCount += 1
}
})
if err != nil {
log.Error("VisitCountMethod GetAccessUserByTime", log.E(err))
return err
}
// 插入数据库
for districtCode, districtValue := range districtMap {
rate := float64(0)
if districtValue != nil && districtValue.RegisterCount > 0 {
rate, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", 100*(float64(districtValue.TodayVisitCount)/float64(districtValue.RegisterCount))), 64)
}
// 更新渠道留存
statFilter := bson.M{
channelmod.FN.ChannelStat.Channel: districtCode,
channelmod.FN.ChannelStat.SumDate: start,
channelmod.FN.ChannelStat.AppId: appid,
}
setDoc := bson.M{
"$set": bson.M{
channelmod.FN.ChannelStat.Day7RetainRate: rate,
},
}
serr := channelmod.UpdateOneChannelStat(ctx, statFilter, setDoc)
if serr != nil {
log.Error("Day1RetainRateMethod UpsertOneDistrictStat ", log.E(err))
return err
}
}
}
return nil
}
skd/job/statdistrict/stat.go
+87
View File
@@ -0,0 +1,87 @@
package statdistrict
import (
"context"
"stat/common/log"
"stat/common/statutils"
"stat/common/utils"
"stat/models/appInfoMod"
statrecordmod "stat/models/statrecord"
"time"
)
/*
* 主要统计任务:
* 1、渠道每天的：直推访问数、裂变访问数、次日留存率、3日留存率、7日留存率。
*/
func RunStat(defaultStatDate time.Time) error {
log.Info("statdistrict stat running...")
ctx := context.Background()
recordMap := defaultTimeMap(defaultStatDate)
// 1.获取统计项上次统计的时候(每天执行)
recoredResult, err := statrecordmod.LastRecordTimeMap(ctx, statrecordmod.DiscStatJob) //第一次从默认记录时间开始统计
if err != nil {
log.Error("statdistrict RunStat lastRecordTime failed!", log.E(err))
return err
}
for _, record := range recoredResult {
recordMap[record.Item] = record.RecordTime
}
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("statdistrict RunStat GetAppIds failed!", log.E(err))
return err
}
statFrq := 24 * time.Hour // 统计频率
curStatTime := utils.BeginOfToday() // 本次统计时间
for item, lastStatAt := range recordMap {
_lastStatAt := lastStatAt
for curStatTime.After(_lastStatAt) {
end := _lastStatAt.Add(statFrq)
// 超过本次统计结束时间，则结束
if end.After(curStatTime) {
end = curStatTime
}
// 根据时间段统计数据
err := runStatMethod(ctx, item, _lastStatAt, end, appIds)
if err != nil {
log.Error("statdistrict statByTimeRange failed!", log.E(err), log.Any("item", item))
return err
}
_lastStatAt = end
}
recordMap[item] = _lastStatAt
}
return nil
}
// runStatMethod 调用统计项方法
func runStatMethod(ctx context.Context, item string, start, end time.Time, appIds []int32) (err error) {
// 1.获取统计方法
statMethod, ok := statMethodMap[item]
if !ok {
return nil
}
// 调用统计方法，获取数据
err = statutils.ServerMon.RunWithName(func() error {
return statMethod(ctx, start, end, appIds)
}, statutils.GetFuncName(statMethod))
if err != nil {
log.Error("statdistrict stat statByTimeRange statMethod failed!", log.Any("statItem", item), log.E(err))
return err
}
// 3.更新记录
if err := statrecordmod.UpsertItemRecord(ctx, statrecordmod.DiscStatJob, item, end); err != nil {
log.Error("statdistrict stat commit UpsertItemRecord failed!", log.E(err))
return err
}
return nil
}
skd/job/stathour/method.go
+260
View File
@@ -0,0 +1,260 @@
package stathour
import (
"context"
"fmt"
"stat/common/enum/consts"
"stat/common/log"
"stat/common/statutils"
"stat/common/utils"
"stat/models/accessmod"
"stat/models/appInfoMod"
downloadinfo "stat/models/downloadinfo"
"stat/models/minutestatmod"
"stat/models/rechargemod"
"stat/models/registermod"
"stat/models/versionMod"
"stat/statg"
"strings"
"time"
)
const (
VisitCount string = "visitCount" // 访问数
NewCount string = "newCount" // 新增数
RechargeCount string = "rechargeCount" // 充值订单数
LandpageClick string = "landpageClick" // 落地页访问次数
)
func defaultTimeMap(defaultStatDate time.Time) map[string]time.Time {
return map[string]time.Time{
RechargeCount: defaultStatDate, // 充值订单数
NewCount: defaultStatDate, // 新增数
VisitCount: defaultStatDate, // 访问数
LandpageClick: defaultStatDate, // 落地页访问次数
}
}
// 获取统计方法
type StatMethod = func(ctx context.Context, start time.Time, end time.Time, appIds []int32) error
// 统计方法汇总
var statMethodMap = map[string]StatMethod{
RechargeCount: RechargeCountMethod, // 充值订单数
NewCount: NewCountMethod, // 新增数
VisitCount: VisitCountMethod, // 访问数
LandpageClick: LandpageClickMethod, // 落地页访问次数
}
// RechargeCountMethod 充值订单数
func RechargeCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
// 获取充值金额
itemList, err := rechargemod.TotalRechargeListV1(ctx, appid, start, end)
if err != nil {
log.Error("DirectAmountMethod DirectRechargeAmount ", log.E(err))
return err
}
stat := &statutils.Stat{}
for _, item := range itemList {
client := statutils.GetClientInfo(item.SysType, item.DevType, item.DistrictCode, item.IsDirect, false, item.IsFormalUser)
stat.Add(item.PayMoney, client)
}
// 更新数据库
doc := &minutestatmod.MinuteStatDoc{
RechargetCount: stat.CountRef(),
RechargetMoney: stat.ValueRef(),
DirectAmount: stat.Total.Direct.ValueRef(),
FissionAmount: &stat.Total.Fission.Value,
DistrictRecharge: &stat.Total.District.Value,
WebRecharge: &stat.Total.Web.Value,
NormalRecharge: &stat.Total.Normal.Value,
AndroidRecharge: stat.Android.ValueRef(),
IosRecharge: stat.Ios.ValueRef(),
LightRecharge: stat.Light.ValueRef(),
H5Recharge: stat.H5.ValueRef(),
}
err = minutestatmod.UpsertMinuteDocStat(ctx, start, appid, doc)
if err != nil {
log.Error("RechargeCountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// NewCountMethod 新增数
func NewCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
stat := &statutils.Stat{}
userDict := make(map[uint64]bool)
err := registermod.RegisterFormalsInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
if item.IsFormal && statutils.IsClientH5OrLight(item.SysType, item.DevType) {
if _, exists := userDict[item.UserId]; !exists {
userDict[item.UserId] = true
client := statutils.GetClientInfo(item.SysType, item.DevType,
item.DistrictCode, item.IsDirect, false, item.IsFormal)
stat.Add(1, client)
}
}
})
if err != nil {
return err
}
err = registermod.RegisterUsersInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
userDict[item.UserId] = true
client := statutils.GetClientInfo(item.SysType, item.DevType,
item.DistrictCode, item.IsDirect, false, item.IsFormal)
stat.Add(1, client)
})
if err != nil {
return err
}
// 更新数据库
doc := &minutestatmod.MinuteStatDoc{
NewCount: stat.Total.CountRef(),
DirectAddCount: stat.Total.Direct.CountRef(),
FissionAddCount: &stat.Total.Fission.Count,
WebAddCount: &stat.Total.Web.Count,
DistrictAddCount: &stat.Total.District.Count,
NormalAddCount: &stat.Total.Normal.Count,
AndroidAddCount: stat.Android.CountRef(),
IosAddCount: stat.Ios.CountRef(),
LightGuestAddCount: stat.Light.Guest.CountRef(),
LightFormalAddCount: stat.Light.Formal.CountRef(),
H5AddCount: stat.H5.CountRef(),
H5FormalAddCount: stat.H5.Formal.CountRef(),
}
err = minutestatmod.UpsertMinuteDocStat(ctx, start, appid, doc)
if err != nil {
log.Error("DirectVisitCountMethod UpsertDailyStat ", log.E(err))
return err
}
iosCount := *stat.Ios.CountRef()
androidCount := *stat.Android.CountRef()
if statg.Conf.RunMod == consts.RunModRelease && appid == 888 {
if iosCount < 5 || androidCount <= 40 {
appinfo, _ := appInfoMod.GetAppInfoById(ctx, appid)
text := fmt.Sprintf("[警告]:应用名称:%s\n10分钟新增异常:\n时间:%s\nandroid:%d\nios:%d\n", appinfo.Name, start, androidCount, iosCount)
err = statg.Bot.SendMonitorMessage(text)
if err != nil {
log.Error("tg Send ", log.E(err))
continue
}
}
}
if statg.Conf.RunMod == consts.RunModRelease && appid == 999 {
if iosCount < 5 || androidCount <= 40 {
appinfo, _ := appInfoMod.GetAppInfoById(ctx, appid)
text := fmt.Sprintf("[警告]:应用名称:%s\n10分钟新增异常:\n时间:%s\nandroid:%d\nios:%d\n", appinfo.Name, start, androidCount, iosCount)
err = statg.Bot.SendMonitorMessage(text)
if err != nil {
log.Error("tg Send ", log.E(err))
continue
}
}
}
if statg.Conf.RunMod == consts.RunModRelease && (appid == 222 || appid == 80) {
if iosCount <= 2 || androidCount <= 10 {
appinfo, _ := appInfoMod.GetAppInfoById(ctx, appid)
text := fmt.Sprintf("[警告]:应用名称:%s\n10分钟新增异常:\n时间:%s\nandroid:%d\nios:%d\n", appinfo.Name, start, androidCount, iosCount)
err = statg.Bot.SendMonitorMessage(text)
if err != nil {
log.Error("tg Send ", log.E(err))
continue
}
}
}
}
return nil
}
// VisitCountMethod 访问数
func VisitCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
start = start.Add(-time.Minute * 10)
end = end.Add(-time.Minute * 10)
for _, appid := range appIds {
stat := &statutils.Stat{}
err := accessmod.VisitInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *accessmod.AccessMod) {
client := statutils.GetClientInfo(item.SysType, item.DevType,
item.DistrictCode, item.IsDirect, false, item.IsFormalUser)
stat.Add(1, client)
})
if err != nil {
log.Error("AppVisitCountMethod:VisitInfoByTimeShard", log.E(err))
return err
}
// 更新数据库
doc := &minutestatmod.MinuteStatDoc{
AccessCount: stat.CountRef(),
DirectAccessCount: stat.Total.Direct.CountRef(),
NaturalAccessCount: stat.Total.Direct.Natural.CountRef(),
FissionAccessCount: stat.Total.Fission.CountRef(),
DistrictAccessCount: stat.Total.District.CountRef(),
WebAccessCount: stat.Total.Web.CountRef(),
NormalAccessCount: stat.Total.Normal.CountRef(),
AndroidAccessCount: stat.Android.CountRef(),
IosAccessCount: stat.Ios.CountRef(),
H5AccessCount: stat.H5.CountRef(),
LightAccessCount: stat.Light.CountRef(),
}
err = minutestatmod.UpsertMinuteDocStat(ctx, start, appid, doc)
if err != nil {
log.Error("DirectVisitCountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
// LandpageClickMethod 落地页访问次数
func LandpageClickMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
downloadList, err := downloadinfo.GetAccessCountByTime(ctx, appid, start, end)
if err != nil {
log.Error("RunMonitor, GetAccessCountByTime", log.E(err))
return err
}
androidClick := int64(0)
appstoreClick := int64(0)
iosAtherClick := int64(0)
h5Click := int64(0)
for _, downloadInfo := range downloadList {
if downloadInfo.SysType == "android" {
androidClick += 1
} else if strings.Contains(downloadInfo.SysType, "ios") || strings.Contains(downloadInfo.SysType, "light") {
if downloadInfo.PackageType == versionMod.Shop {
appstoreClick += 1
} else if downloadInfo.PackageType == versionMod.Enterprise {
iosAtherClick += 1
} else if downloadInfo.PackageType == versionMod.Light {
h5Click += 1
}
}
}
// 更新数据库
doc := &minutestatmod.MinuteStatDoc{
LandpageAndroidClick: &androidClick,
LandpageIosAppstoreClick: &appstoreClick,
LandpageIosAthorClick: &iosAtherClick,
LandpageIosH5Click: &h5Click,
}
err = minutestatmod.UpsertMinuteDocStat(ctx, start, appid, doc)
if err != nil {
log.Error("DirectVisitCountMethod UpsertDailyStat ", log.E(err))
return err
}
}
return nil
}
skd/job/stathour/stat.go
+94
View File
@@ -0,0 +1,94 @@
package stathour
import (
"context"
"stat/common/log"
"stat/common/statutils"
"stat/common/utils"
"stat/models/appInfoMod"
statrecordmod "stat/models/statrecord"
"time"
)
/*
* 主要统计任务:
* 按1个小时一个点，记录实时数据 app（新增、访问数....）
*/
// RunStat // 按10分钟的频率去统计
func RunStat(defaultStatDate time.Time) error {
log.Info("APP hour stat ......")
ctx := context.Background()
recordMap := defaultTimeMap(defaultStatDate)
// 1.获取统计项上次统计的时候(每天执行)
recoredResult, err := statrecordmod.LastRecordTimeMap(ctx, statrecordmod.AppHour) //第一次从默认记录时间开始统计
if err != nil {
log.Error("APP hour stat ", log.E(err))
return err
}
for _, record := range recoredResult {
recordMap[record.Item] = record.RecordTime
}
now := time.Now().Add(time.Minute * -5) // 延迟5分钟，数据同步完成
statFrq := 10 * time.Minute // 统计频率
curStatTime := utils.RecentMinute(now, 10) // 按10分钟对齐
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("APP hour stat GetAppIds failed!", log.E(err))
return err
}
for item, lastStatAt := range recordMap {
_lastStatAt := lastStatAt
for curStatTime.After(_lastStatAt) {
end := _lastStatAt.Add(statFrq)
// 超过本次统计结束时间，则结束
if end.After(curStatTime) {
end = curStatTime
}
// 根据时间段统计数据
//fmt.Println("APP hour stat ", _lastStatAt, end)
err := runStatMethod(ctx, item, _lastStatAt, end, appIds)
if err != nil {
log.Error("APP hour stat runStatMethod failed!", log.E(err), log.Any("item", item))
return err
}
_lastStatAt = end
}
recordMap[item] = _lastStatAt
}
log.Info("APP hour stat ")
return nil
}
// runStatMethod 调用统计项方法
func runStatMethod(ctx context.Context, item string, start, end time.Time, appIds []int32) (err error) {
// 1.获取统计方法
statMethod, ok := statMethodMap[item]
if !ok {
return nil
}
// 调用统计方法，获取数据
err = statutils.ServerMon.RunWithName(func() error {
return statMethod(ctx, start, end, appIds)
}, statutils.GetFuncName(statMethod))
if err != nil {
log.Error("APP hour stat statMethod failed!", log.Any("statItem", item), log.E(err))
return err
}
// 3.更新记录
if err := statrecordmod.UpsertItemRecord(ctx, statrecordmod.AppHour, item, end); err != nil {
log.Error("APP hour stat commit UpsertItemRecord failed!", log.E(err))
return err
}
return nil
}
skd/job/statlandpage/stat.go
+92
View File
@@ -0,0 +1,92 @@
package statlandpage
import (
"context"
"fmt"
"stat/common/log"
"stat/common/statutils"
"stat/common/utils"
statrecordmod "stat/models/statrecord"
"time"
)
type StatMethod func(ctx context.Context, start, end time.Time) error
type TimeAlignFunc func(t time.Time) time.Time
func timeAlignForDaily(t time.Time) time.Time {
return utils.BeginWithTime(t)
}
func timeAlignForHour(t time.Time) time.Time {
return utils.RecentMinute(t, 60)
}
func RunStatHour(defaultStatDate time.Time) error {
ctx := context.Background()
job := statrecordmod.LandPageHourStatJob
recordMap := defaultTimeMap(defaultStatDate)
return recordStat(ctx, recordMap, job, time.Hour, -time.Hour, statLandPageHourApps, timeAlignForHour)
}
func RunStatDaily(defaultStatDate time.Time) error {
ctx := context.Background()
job := statrecordmod.LandPageDailyStatJob
recordMap := defaultTimeMap(defaultStatDate)
return recordStat(ctx, recordMap, job, 24*time.Hour, -time.Hour, statLandPageDailyApps, timeAlignForDaily)
}
// recordStat 统计
func recordStat(ctx context.Context, recordMap map[string]time.Time, job string, timeStep, timeBack time.Duration, stat StatMethod, align TimeAlignFunc) error {
recordResult, err := statrecordmod.LastRecordTimeMap(ctx, job) //第一次从默认记录时间开始统计
if err != nil {
log.Error("LastRecordTimeMap", log.E(err))
return err
}
for _, record := range recordResult {
recordMap[record.Item] = record.RecordTime
}
now := time.Now().Add(timeBack)
curStatTime := align(now)
for item, lastStatAt := range recordMap {
_lastStatAt := align(lastStatAt)
for curStatTime.After(_lastStatAt) {
end := _lastStatAt.Add(timeStep)
fmt.Printf("item:%s, start:%s, end:%s\n", item, _lastStatAt, end)
err := runStatMethod(ctx, item, job, _lastStatAt, end, stat)
if err != nil {
log.Error("recordStat failed!", log.E(err), log.Any("item", item), log.Any("job", job))
return err
}
_lastStatAt = end
}
recordMap[item] = _lastStatAt
}
return err
}
// runStatMethod 调用统计项方法
func runStatMethod(ctx context.Context, item, job string, start, end time.Time, stat StatMethod) error {
var err error
// 调用统计方法，获取数据
err = statutils.ServerMon.RunWithName(func() error {
return stat(ctx, start, end)
}, statutils.GetFuncName(stat))
if err != nil {
log.Error("runStatMethod failed!", log.Any("statItem", item), log.E(err))
return err
}
if err := statrecordmod.UpsertItemRecord(ctx, job, item, end); err != nil {
log.Error("commit UpsertItemRecord failed!", log.E(err))
return err
}
return nil
}
skd/job/statlandpage/stat_test.go
+18
View File
@@ -0,0 +1,18 @@
package statlandpage
import (
"stat/common/utils"
"stat/tool/pubtest"
"testing"
"time"
)
func TestRunStat(t *testing.T) {
pubtest.TestInit("/Users/mac/Works/codes/stat_center/run/zhihu-test")
s, _ := time.Parse(time.RFC3339, "2023-06-15T15:00:00+08:00")
//curStatTime := utils.RecentMinute(s, 60) // 按60分钟对齐
curStatTime := utils.BeginWithTime(s) // 按60分钟对齐
//fmt.Printf("%s", curStatTime)
RunStatDaily(curStatTime)
}
skd/job/statlandpage/statlandpage.go
+134
View File
@@ -0,0 +1,134 @@
package statlandpage
import (
"context"
"stat/common/log"
"stat/common/utils"
"stat/models/appInfoMod"
"stat/models/downloadCpc"
"stat/models/downloadinfo"
"stat/models/downloadstat"
"stat/models/registermod"
"time"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/bson"
)
const (
LandPageStat string = "landPageStat" // 落地页统计
)
func defaultTimeMap(defaultStatDate time.Time) map[string]time.Time {
return map[string]time.Time{
LandPageStat: defaultStatDate, // 落地页统计
}
}
func statLandPageDailyApps(ctx context.Context, start, end time.Time) error {
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("GetAppIds failed!", log.E(err))
return err
}
for _, id := range appIds {
if err := statLandPage(ctx, start, end, id, downloadstat.StatTypeDaily); err != nil {
return err
}
}
time.Sleep(3 * time.Second)
return nil
}
func statLandPageHourApps(ctx context.Context, start, end time.Time) error {
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("GetAppIds failed!", log.E(err))
return err
}
for _, id := range appIds {
if err := statLandPage(ctx, start, end, id, downloadstat.StatTypeHour); err != nil {
return err
}
}
return nil
}
func statLandPage(ctx context.Context, start, end time.Time, appId int32, statType downloadstat.StatType) error {
old, err := downloadstat.FindOneStat(ctx, bson.M{
"appId": appId,
"type": statType,
"sumDate": start,
})
if err != nil {
if err != mongo.ErrNoDocuments {
return err
}
}
if old != nil {
return nil // 已经存在
}
visitTotal := int64(0)
visitDict := make(map[string]int64)
visitList, err := downloadCpc.FindManyDownloadCpc(ctx, bson.M{
"appId": appId,
"createdAt": bson.M{"$gte": start, "$lt": end},
})
if err != nil {
return err
}
for _, visit := range visitList {
visitTotal++
visitDict[visit.IP]++
}
downTotal := int64(0)
downDict := make(map[string]int64)
downDictEx := make(map[string]int64)
downList, err := downloadinfo.FindManyDownloadInfo(ctx, bson.M{
"appId": appId,
"createdAt": bson.M{"$gte": start.Add(-time.Hour), "$lt": end},
})
if err != nil {
return err
}
for _, down := range downList {
downDictEx[down.IP]++
if !down.CreatedAt.Before(start) && down.CreatedAt.Before(end) {
downTotal++
downDict[down.IP]++
}
}
registerDict := make(map[string]int64)
var registerCount int64
err = registermod.RegisterUsersInfoByTimeShard(ctx, appId, start, end, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
if _, exists := downDictEx[item.IP]; exists {
registerDict[item.IP]++
registerCount++ // 这里只计算来自落地页的注册
}
})
if err != nil {
return err
}
stat := &downloadstat.Stat{
ID: primitive.NewObjectID(),
AppId: appId,
SumDate: start,
EndDate: end,
Hour: start.Hour(),
Type: statType,
VisitTotal: visitTotal,
VisitUnique: int64(len(visitDict)),
DownTotal: downTotal,
DownUnique: int64(len(downDict)),
InstallTotal: registerCount, // 落地页注册
InstallUnique: int64(len(registerDict)),
UpdatedAt: time.Now(),
CreatedAt: time.Now(),
}
return downloadstat.InsertOneStat(ctx, stat)
}
skd/job/statminute/method.go
+602
View File
@@ -0,0 +1,602 @@
package statminute
import (
"context"
"fmt"
"math/rand"
"stat/common/log"
"stat/common/statutils"
"stat/common/topicmessage"
"stat/common/utils"
"stat/models/accessmod"
"stat/models/channelmod"
"stat/models/downloadinfo"
"stat/models/rechargegamemod"
"stat/models/rechargemod"
"stat/models/registermod"
"stat/models/syncvip"
"stat/models/versionMod"
"stat/models/withdrawGameOrdermod"
"strings"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
const (
ChannelStat string = "visitInfo" // 渠道统计
)
func defaultTimeMap(defaultStatDate time.Time) map[string]time.Time {
return map[string]time.Time{
ChannelStat: defaultStatDate, // 渠道统计
}
}
// 获取统计方法
type StatMethod = func(ctx context.Context, start time.Time, end time.Time, appIds []int32) error
// 统计方法汇总
var statMethodMap = map[string]StatMethod{
ChannelStat: ChannelSettlementMethod, // 渠道统计
}
func canUserDeduction(ruleInfo *channelmod.ChannelDeductionRule, index int, incomeRate float64, isDirect bool) bool {
rate := int32(0) // 扣量比例
if ruleInfo.Settlement == channelmod.SettlementCPA || ruleInfo.Settlement == channelmod.SettlementCPS {
//自动扣量
if ruleInfo.Settlement == channelmod.SettlementCPA && ruleInfo.AutoDeduction && incomeRate > 0.0 && ruleInfo.LessIncomeRate > incomeRate {
rate = 80
} else {
if index < 1 { // 一阶
rate = ruleInfo.First
} else if index <= 3 { // 二阶
rate = ruleInfo.Second
} else if index <= 6 { // 三阶
rate = ruleInfo.Third
} else if index <= 10 { // 四阶
rate = ruleInfo.Fourth
} else {
rate = ruleInfo.Fifth
}
}
if isDirect && int32(rand.Intn(100)) < rate { // rate扣量比例
return true
}
}
return false
}
func deductionCountMethod(ctx context.Context, ruleInfo *channelmod.ChannelDeductionRule, settlement *channelmod.ChannelSettlement, registerList []*registermod.RegisterMod, incomeRate float64) error {
deductionUserIds := make([]int64, 0)
// 先计算扣量
for index, item := range registerList {
item.IsDeduction = false
if canUserDeduction(ruleInfo, index, incomeRate, item.IsDirect) {
item.IsDeduction = true
deductionUserIds = append(deductionUserIds, int64(item.UserId))
}
}
// 统计IP
ipDict := make(map[string]int64)
for _, item := range registerList {
if item.IP != "" {
ipDict[item.IP]++
}
}
// 1.统计这段时间内的注册用户
stat := &statutils.Stat{}
uniqueStat := &statutils.Stat{}
for _, item := range registerList {
client := statutils.GetClientInfo(item.SysType, item.DevType, item.DistrictCode, item.IsDirect, item.IsDeduction, item.IsFormal)
stat.Add(1, client)
if _, exists := ipDict[item.IP]; exists {
uniqueStat.Add(1, client)
}
}
settlement.NewTotalIpCount += uniqueStat.Count()
settlement.NewTotalCount += stat.Total.Count()
settlement.NewDirectCount += stat.Total.Direct.Count()
settlement.NewFissionCount += stat.Total.Fission.Count
settlement.NewDeductionCount += stat.Total.Deduction.Count
settlement.NewLightCount += stat.Light.Count()
settlement.NewLightDirectCount += stat.Light.Total.Direct.Count()
settlement.NewLightFissionCount += stat.Light.Total.Fission.Count
settlement.NewLightDeductionCount += stat.Light.Total.Deduction.Count
settlement.NewLightNaturalCount += stat.Light.Total.Direct.Natural.Count
settlement.NewLightGuestCount += stat.Light.Guest.Count()
settlement.NewLightGuestDirectCount += stat.Light.Guest.Direct.Count()
settlement.NewLightGuestFissionCount += stat.Light.Guest.Fission.Count
settlement.NewLightGuestDeductionCount += stat.Light.Guest.Deduction.Count
settlement.NewLightFormalCount += stat.Light.Formal.Count()
settlement.NewLightFormalDirectCount += stat.Light.Formal.Direct.Count()
settlement.NewLightFormalFissionCount += stat.Light.Formal.Fission.Count
settlement.NewLightFormalDeductionCount += stat.Light.Formal.Deduction.Count
settlement.NewAndroidCount += stat.Android.Count()
settlement.NewAndroidDirectCount += stat.Android.Direct.Count()
settlement.NewAndroidFissionCount += stat.Android.Fission.Count
settlement.NewAndroidDeductionCount += stat.Android.Deduction.Count
settlement.NewIosCount += stat.Ios.Count()
settlement.NewIosDirectCount += stat.Ios.Direct.Count()
settlement.NewIosFissionCount += stat.Ios.Fission.Count
settlement.NewIosDeductionCount += stat.Ios.Deduction.Count
settlement.NewH5Count += stat.H5.Count()
settlement.NewH5DirectCount += stat.H5.Total.Direct.Count()
settlement.NewH5FissionCount += stat.H5.Total.Fission.Count
settlement.NewH5DeductionCount += stat.H5.Total.Deduction.Count
settlement.NewH5AndroidCount += stat.H5.Android.Count()
settlement.NewH5IosCount += stat.H5.Ios.Count()
settlement.NewH5PcCount += stat.H5.Pc.Count()
settlement.NewH5FormalCount += stat.H5.Formal.Count()
settlement.NewH5FormalDirectCount += stat.H5.Formal.Direct.Count()
settlement.NewH5FormalFissionCount += stat.H5.Formal.Fission.Count
settlement.NewH5FormalDeductionCount += stat.H5.Formal.Deduction.Count
// 5 更新扣量用户
if len(deductionUserIds) > 0 {
setdoc := bson.M{"$set": bson.M{"isDeduction": true}}
filter := bson.M{
"appId": ruleInfo.AppId,
"userId": bson.M{"$in": deductionUserIds},
}
err := registermod.UpdateManyRegister(ctx, filter, setdoc)
if err != nil {
log.Error("NewCountMethod UpdateManyRegister ", log.E(err))
return err
}
}
return nil
}
func channelChannelStatMethod(ctx context.Context, ruleInfo *channelmod.ChannelDeductionRule, settlement *channelmod.ChannelSettlement, visitList accessmod.VisitInfoList,
start, end time.Time, appid int32) error {
stat := &statutils.Stat{}
for _, item := range visitList {
client := statutils.GetClientInfo(item.SysType, item.DevType, item.DistrictCode, item.IsDirect, false, item.IsFormalUser)
stat.Add(1, client)
}
settlement.TotalVisitCount = stat.Count()
settlement.DirectVisitCount = stat.Total.Direct.Count()
settlement.FissionVisitCount = stat.Total.Fission.Count
settlement.AndroidVisitCount = stat.Android.Count()
settlement.IosVisitCount = stat.Ios.Count()
settlement.LightVisitCount = stat.Light.Count()
settlement.LightVisitDirectCount = stat.Light.Total.Direct.Count()
settlement.LightVisitFissionCount = stat.Light.Total.Fission.Count
settlement.LightVisitDeductionCount = stat.Light.Total.Deduction.Count
settlement.LightVisitNaturalCount = stat.Light.Total.Direct.Natural.Count
settlement.H5VisitCount = stat.H5.Count()
settlement.H5AndroidVisitCount = stat.H5.Android.Count()
settlement.H5IosVisitCount = stat.H5.Ios.Count()
settlement.H5PcVisitCount = stat.H5.Pc.Count()
return nil
}
func channelVipMethod(ctx context.Context, ruleInfo *channelmod.ChannelDeductionRule, settlement *channelmod.ChannelSettlement, VipList syncvip.SyncVipList,
start, end time.Time, appid int32) error {
deductionVipOrderIds := make([]primitive.ObjectID, 0)
for _, buyInfo := range VipList {
// 统计渠道信息
if buyInfo.IsDirect && buyInfo.BuyVIPType == topicmessage.BuyVIPTypeMoney {
if ruleInfo.Settlement == channelmod.SettlementCPS {
if ruleInfo.TodayTime != utils.BeginWithTime(start) {
ruleInfo.TodayDeduction = 0
ruleInfo.TodayTime = utils.BeginWithTime(start)
}
ruleInfo.TodayDeduction += buyInfo.Money
if ruleInfo.CPSBegin < ruleInfo.TodayDeduction {
if int32(rand.Intn(100)) < int32(ruleInfo.CPSDeductionRate) {
deductionVipOrderIds = append(deductionVipOrderIds, buyInfo.ID)
settlement.VipPayDeduction += buyInfo.Money
}
}
}
}
// 统计信息
if buyInfo.IsDirect {
if buyInfo.BuyVIPType == topicmessage.BuyVIPTypeMoney {
settlement.VipPayMoney += buyInfo.Money
} else {
settlement.VipPayCoin += buyInfo.Amount
}
}
}
if len(deductionVipOrderIds) > 0 {
settlement.VipPayMoney = settlement.VipPayMoney - settlement.VipPayDeduction
setdoc := bson.M{"$set": bson.M{syncvip.FN.SyncVip.IsDeduction: true}}
filter := bson.M{
syncvip.FN.SyncVip.ID: bson.M{"$in": deductionVipOrderIds},
syncvip.FN.SyncVip.AppId: ruleInfo.AppId,
}
err := syncvip.UpdateOneSyncVip(ctx, filter, setdoc)
if err != nil {
log.Error("channelRechargeCountMethod UpdateManyRecharge ", log.E(err))
return err
}
}
return nil
}
func channelRechargeCountMethod(ctx context.Context, ruleInfo *channelmod.ChannelDeductionRule, settlement *channelmod.ChannelSettlement, rechargeList rechargemod.RechargeInfoList,
start, end time.Time, appid int32) error {
stat := &statutils.Stat{}
statNew := statutils.Stat{}
for _, item := range rechargeList {
client := statutils.GetClientInfo(item.SysType, item.DevType, item.DistrictCode, item.IsDirect, false, item.IsFormalUser)
stat.Add(item.Money, client)
if item.RegisterTime.After(item.SuccessAt.AddDate(0, 0, -1)) {
statNew.Add(item.Money, client)
}
}
settlement.TotalAmount = stat.Value()
settlement.DirectAmount = stat.Total.Direct.Value()
settlement.DirectAmountCount = stat.Total.Direct.Count()
settlement.FissionAmount = stat.Total.Fission.Value
settlement.FissionAmountCount = stat.Total.Fission.Count
settlement.NaturalAmount = stat.Total.Direct.Natural.Value
settlement.NaturalAmountCount = stat.Total.Direct.Natural.Count
settlement.DeductionAmount = stat.Total.Deduction.Value
settlement.DeductionAmountCount = stat.Total.Deduction.Count
settlement.AndroidAmount = stat.Android.Value()
settlement.AndroidAmountCount = stat.Android.Count()
settlement.IosAmount = stat.Ios.Value()
settlement.IosAmountCount = stat.Ios.Count()
settlement.H5Amount = stat.H5.Value()
settlement.H5AmountCount = stat.H5.Count()
settlement.LightAmount = stat.Light.Value()
settlement.LightAmountCount = stat.Light.Count()
settlement.LightDirectAmount = stat.Light.Total.Direct.Value()
settlement.LightDirectAmountCount = stat.Light.Total.Direct.Count()
settlement.LightFissionAmount = stat.Light.Total.Fission.Value
settlement.LightFissionAmountCount = stat.Light.Total.Fission.Count
settlement.LightDeductionAmount = stat.Light.Total.Deduction.Value
settlement.LightDeductionAmountCount = stat.Light.Total.Deduction.Count
settlement.LightNaturalAmount = stat.Light.Total.Direct.Natural.Value
settlement.LightNaturalAmountCount = stat.Light.Total.Direct.Natural.Count
settlement.NewRechargeAmount = statNew.Value()
return nil
}
// GameRechargeMethod 游戏充值
func channelGameRechargeMethod(ctx context.Context, ruleInfo *channelmod.ChannelDeductionRule,
rechageList rechargegamemod.GameRechargeInfoList, withdrawList []withdrawGameOrdermod.GameWithdrawOrder,
settlement *channelmod.ChannelSettlement,
start, end time.Time, appid int32) error {
for _, withdrawInfo := range withdrawList {
if ruleInfo.Channel == withdrawInfo.DistrictCode {
settlement.GameWithdrawAmount += withdrawInfo.Money
}
}
for _, rechageInfo := range rechageList {
if ruleInfo.Channel == rechageInfo.DistrictCode {
settlement.GameRechageAmount += rechageInfo.Money
}
}
return nil
}
// channelLandPageMethod 落地页访问
func channelLandPageMethod(ctx context.Context, settlement *channelmod.ChannelSettlement, start, end time.Time, appid int32) error {
itemList, err := downloadinfo.GetChannelDownloadPageListByTime(ctx, settlement.Channel, appid, start, end)
if err != nil {
return err
}
androidClick := int64(0)
appstoreClick := int64(0)
iosAtherClick := int64(0)
lightClick := int64(0)
for _, downloadInfo := range itemList {
if downloadInfo.SysType == "android" {
androidClick += 1
} else if strings.Contains(downloadInfo.SysType, "ios") || strings.Contains(downloadInfo.SysType, "light") {
if downloadInfo.PackageType == versionMod.Shop {
appstoreClick += 1
} else if downloadInfo.PackageType == versionMod.Enterprise {
iosAtherClick += 1
} else if downloadInfo.PackageType == versionMod.Light {
lightClick += 1
}
}
}
settlement.LandPageVisitCount = int64(len(itemList))
settlement.LandpageAndroidClick = androidClick
settlement.LandpageIosAppstoreClick = appstoreClick
settlement.LandpageIosAthorClick = iosAtherClick
settlement.LandpageIosH5Click = lightClick
return nil
}
// ChannelSettlementMethod 渠道计算
func ChannelSettlementMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
for _, appid := range appIds {
channelDeductionRules, err := channelmod.FindManyChannelDeductionRule(ctx, bson.M{channelmod.FN.ChannelDeductionRule.AppId: appid})
if err != nil {
log.Error("ChannelSettlementMethod FindManyChannelDeductionRule", log.E(err))
return err
}
channnels := make([]string, 0)
for _, channnelInfo := range channelDeductionRules {
channnels = append(channnels, channnelInfo.Channel)
}
channelInfolist, err := channelmod.FindManyChannelInfo(ctx, bson.M{channelmod.FN.ChannelInfo.Channel: bson.M{"$in": channnels}})
if err != nil {
log.Error("ChannelSettlementMethod FindManyChannelInfo", log.E(err))
return err
}
registerList, err := registermod.RegisterUsersInfoByTime(ctx, appid, start, end)
if err != nil {
log.Error("NewCountMethod RegisterUsersInfoByDistrict ", log.E(err))
return err
}
registerFormalList, err := registermod.RegisterFormalsInfoByTime(ctx, appid, start, end)
if err != nil {
log.Error("NewCountMethod RegisterUsersInfoByDistrict ", log.E(err))
return err
}
vipfilter := bson.M{
syncvip.FN.SyncVip.AppId: appid,
syncvip.FN.SyncVip.SuccessAt: bson.M{"$gte": start, "$lt": end},
}
VipList, err := syncvip.FindManySyncVip(ctx, vipfilter)
if err != nil {
log.Error("ChannelChannelStatMethod FindManyVisitInfo ", log.E(err))
return err
}
visitFilter := bson.M{"appId": appid, "visitAt": bson.M{"$gte": start, "$lt": end}}
visitList, _ := accessmod.FindManyVisitInfo(ctx, visitFilter)
rechargeFilter := bson.M{"appId": appid, "successAt": bson.M{"$gte": start, "$lt": end}}
rechargeList, _ := rechargemod.FindManyRechargeInfo(ctx, rechargeFilter)
for _, ruleInfo := range channelDeductionRules {
if ruleInfo.CreatedAt.After(start) {
continue
}
//fmt.Println("channelDeductionRules", ruleInfo.Channel)
settlement := &channelmod.ChannelSettlement{
SumDate: start,
AppId: appid,
AppType: ruleInfo.AppType,
PromoteMode: ruleInfo.PromoteMode,
Channel: ruleInfo.Channel,
}
for _, channelInfo := range channelInfolist {
if channelInfo.Channel == ruleInfo.Channel {
settlement.Type = channelInfo.Type
settlement.ParentChannel = channelInfo.ParentChannel
settlement.UserName = channelInfo.UserName
}
}
if ruleInfo.SettlementTime == start {
// 已经结算
continue
}
statFilter := bson.M{
channelmod.FN.ChannelStat.Channel: ruleInfo.Channel,
channelmod.FN.ChannelStat.SumDate: utils.BeginWithTime(start),
channelmod.FN.ChannelStat.AppId: appid,
}
channelStatInfo, err := channelmod.FindOneChannelStat(ctx, statFilter)
if err != nil {
log.Error("ChannelSettlementMethod CountChannelStats", log.E(err))
return err
}
// 统计新增情况
userDict := make(map[uint64]bool)
channelRegisterList := make([]*registermod.RegisterMod, 0)
for _, v := range registerFormalList {
if v.IsFormal && statutils.IsClientH5OrLight(v.SysType, v.DevType) {
if v.DistrictCode == ruleInfo.Channel {
userDict[v.UserId] = true
channelRegisterList = append(channelRegisterList, v)
}
}
}
for _, v := range registerList {
if v.DistrictCode == ruleInfo.Channel {
if _, exists := userDict[v.UserId]; !exists {
userDict[v.UserId] = true
channelRegisterList = append(channelRegisterList, v)
}
}
}
oldIncomeRate := float64(0)
if channelStatInfo != nil {
oldIncomeRate = channelStatInfo.IncomeRate
}
deductionCountMethod(ctx, ruleInfo, settlement, channelRegisterList, oldIncomeRate)
// 统计访问量
channelAccessList := make(accessmod.VisitInfoList, 0)
for _, v := range visitList {
if v.DistrictCode == ruleInfo.Channel {
channelAccessList = append(channelAccessList, v)
}
}
channelChannelStatMethod(ctx, ruleInfo, settlement, channelAccessList, start, end, appid)
// 统计充值数据
channelRechargeList := make(rechargemod.RechargeInfoList, 0)
for _, v := range rechargeList {
if v.DistrictCode == ruleInfo.Channel {
channelRechargeList = append(channelRechargeList, v)
}
}
channelRechargeCountMethod(ctx, ruleInfo, settlement, channelRechargeList, start, end, appid)
// 统计VIP购买
channelVipList := make(syncvip.SyncVipList, 0)
for _, v := range VipList {
if v.DistrictCode == ruleInfo.Channel {
channelVipList = append(channelVipList, v)
}
}
channelVipMethod(ctx, ruleInfo, settlement, channelVipList, start, end, appid)
channelLandPageMethod(ctx, settlement, start, end, appid)
// 更新渠道信息
ruleFilter := bson.M{channelmod.FN.ChannelDeductionRule.ID: ruleInfo.ID}
ruleSet := bson.M{"$set": bson.M{
channelmod.FN.ChannelDeductionRule.SettlementTime: start,
channelmod.FN.ChannelDeductionRule.TodayTime: ruleInfo.TodayTime,
channelmod.FN.ChannelDeductionRule.TodayDeduction: ruleInfo.TodayDeduction,
}}
err = channelmod.UpdateManyChannelDeductionRule(ctx, ruleFilter, ruleSet)
if err != nil {
log.Error("ChannelSettlementMethod UpdateManyChannelDeductionRule", log.E(err))
return err
}
settlement.CorrectValidData()
//插入数据
err = channelmod.InsertOneChannelSettlement(ctx, settlement)
if err != nil {
log.Error("ChannelSettlementMethod InsertOneChannelSettlement", log.E(err))
return err
}
if channelStatInfo.Channel != ruleInfo.Channel {
channelStatInfo.Channel = ruleInfo.Channel
channelStatInfo.SumDate = utils.BeginWithTime(start)
channelStatInfo.AppId = appid
channelStatInfo.Type = settlement.Type
channelStatInfo.UserName = settlement.UserName
channelStatInfo.Settlement = ruleInfo.Settlement
channelStatInfo.CPAPrice = ruleInfo.CPAPrice
channelStatInfo.CPSRate = ruleInfo.CPSRate
channelStatInfo.AppType = ruleInfo.AppType
channelStatInfo.PromoteMode = ruleInfo.PromoteMode
channelStatInfo.ParentChannel = settlement.ParentChannel
channelStatInfo.IsPay = false
// 数据赋值
settlement.AssignmentTo(channelStatInfo)
// 插入记录
err = channelmod.InsertOneChannelStat(ctx, channelStatInfo)
if err != nil {
log.Error("ChannelSettlementMethod InsertOneChannelStat", log.E(err))
return err
}
} else {
// 数据累加
settlement.AccumulateTo(channelStatInfo)
// 更新记录
setdoc := bson.M{"$set": channelStatInfo}
err = channelmod.UpdateOneChannelStat(ctx, statFilter, setdoc)
if err != nil {
log.Error("ChannelSettlementMethod UpdateOneChannelStat", log.E(err))
return err
}
}
}
}
return nil
}
// 更新渠道数据
func ChannelSettlementUpdateByHour() error {
ctx := context.Background()
nowHour := utils.BeginOfHour()
statFilter := bson.M{
channelmod.FN.ChannelStat.SumDate: utils.BeginWithTime(nowHour.Add(-30 * time.Minute)), // 确保:0点是获取上一天数据
"$or": bson.A{
bson.M{channelmod.FN.ChannelStat.NewDirectCount: bson.M{"$gt": 0}},
bson.M{channelmod.FN.ChannelStat.DirectAmount: bson.M{"$gt": 0}},
},
}
statList, err := channelmod.FindManyChannelStat(ctx, statFilter)
if err != nil {
log.Error("ChannelSettlementUpdateByHour FindManyChannelStat", log.E(err))
return err
}
yesterdayFilter := bson.M{
channelmod.FN.ChannelStatYestoday.SumDate: nowHour.AddDate(0, 0, -1),
}
statYesterdayList, _ := channelmod.FindManyChannelStatYestoday(ctx, yesterdayFilter)
for _, channelStatInfo := range statList {
channelStatInfoYesterday := &channelmod.ChannelStatYesterday{
// base
Channel: channelStatInfo.Channel,
SumDate: nowHour,
AppId: channelStatInfo.AppId,
ParentChannel: channelStatInfo.ParentChannel,
Settlement: channelStatInfo.Settlement,
UpdatedAt: time.Now(),
CreatedAt: time.Now(),
// new
NewCount: channelStatInfo.NewTotalCount,
NewDirectCount: channelStatInfo.NewDirectCount,
NewFissionCount: channelStatInfo.NewFissionCount,
NewAndroidCount: channelStatInfo.NewAndroidCount,
NewIosCount: channelStatInfo.NewIosCount,
NewH5Count: channelStatInfo.NewH5Count,
NewH5FormalCount: channelStatInfo.NewH5FormalCount,
NewLightGuestCount: channelStatInfo.NewLightGuestCount,
NewLightFormalCount: channelStatInfo.NewLightFormalCount,
// recharge
RechargeAmount: channelStatInfo.TotalAmount,
DirectAmount: channelStatInfo.DirectAmount,
FissionAmount: channelStatInfo.FissionAmount,
AndroidRechargeAmount: channelStatInfo.AndroidAmount,
IosRechargeAmount: channelStatInfo.IosAmount,
LightRechargeAmount: channelStatInfo.LightAmount,
H5RechargeAmount: channelStatInfo.H5Amount,
// other
VipPayCoin: channelStatInfo.VipPayCoin,
VipPayMoney: channelStatInfo.VipPayMoney,
VipPayDeduction: channelStatInfo.VipPayDeduction,
}
for _, yesterdayStatInfo := range statYesterdayList {
// 获取昨天数据
if yesterdayStatInfo.Channel == channelStatInfo.Channel && yesterdayStatInfo.AppId == channelStatInfo.AppId {
newYesterday := yesterdayStatInfo.NewCount
newToday := channelStatInfo.NewTotalCount
channelStatInfoYesterday.NewIncreaseCount = newToday - newYesterday
if newYesterday != 0 {
channelStatInfoYesterday.NewIncreaseRate = fmt.Sprintf("%.2f", float64(newToday-newYesterday)*100/float64(newYesterday)) + "%"
}
rechargeYesterday := yesterdayStatInfo.RechargeAmount
rechargeToday := channelStatInfo.TotalAmount
channelStatInfoYesterday.RechargeIncreaseAmount = rechargeToday - rechargeYesterday
if rechargeYesterday != 0 {
channelStatInfoYesterday.RechargeIncreaseRate = fmt.Sprintf("%.2f", float64(rechargeToday-rechargeYesterday)*100/float64(rechargeYesterday)) + "%"
}
}
}
err := channelmod.InsertOneChannelStatYestoday(ctx, channelStatInfoYesterday)
if err != nil {
log.Error("ChannelSettlementMethod InsertOneChannelStatYestoday", log.E(err))
return err
}
}
return nil
}
skd/job/statminute/stat.go
+95
View File
@@ -0,0 +1,95 @@
package statminute
import (
"context"
"fmt"
"stat/common/log"
"stat/common/statutils"
"stat/common/utils"
"stat/models/appInfoMod"
statrecordmod "stat/models/statrecord"
"time"
)
/*
* 主要统计任务:
* 按5分钟一个点，记录实时数据 app（新增、访问数....）。（曲线图）
*/
// RunStat 任务1:统计每天日活
func RunStat(defaultStatDate time.Time) error {
log.Info("channel deduction ......")
ctx := context.Background()
recordMap := defaultTimeMap(defaultStatDate)
// 1.获取统计项上次统计的时候(每天执行)
recoredResult, err := statrecordmod.LastRecordTimeMap(ctx, statrecordmod.ChannelDeduction) //第一次从默认记录时间开始统计
if err != nil {
log.Error("channel deduction ", log.E(err))
return err
}
for _, record := range recoredResult {
recordMap[record.Item] = record.RecordTime
}
now := time.Now().Add(time.Minute * -5) // 延迟2分钟，数据同步完成
statFrq := 15 * time.Minute // 统计频率
curStatTime := utils.RecentMinute(now, 15) // 按15分钟对齐
appIds, err := appInfoMod.GetAppIds(ctx)
if err != nil {
log.Error("channel deduction GetAppIds failed!", log.E(err))
return err
}
for item, lastStatAt := range recordMap {
_lastStatAt := lastStatAt
for curStatTime.After(_lastStatAt) {
end := _lastStatAt.Add(statFrq)
// 超过本次统计结束时间，则结束
if end.After(curStatTime) {
end = curStatTime
}
// 根据时间段统计数据
fmt.Println("channel deduction ", _lastStatAt, end)
err := runStatMethod(ctx, item, _lastStatAt, end, appIds)
if err != nil {
log.Error("channel deduction runStatMethod failed!", log.E(err), log.Any("item", item))
return err
}
_lastStatAt = end
}
recordMap[item] = _lastStatAt
}
log.Info("channel deduction ")
return err
}
// runStatMethod 调用统计项方法
func runStatMethod(ctx context.Context, item string, start, end time.Time, appIds []int32) (err error) {
// 1.获取统计方法
statMethod, ok := statMethodMap[item]
if !ok {
return nil
}
// 调用统计方法，获取数据
err = statutils.ServerMon.RunWithName(func() error {
return statMethod(ctx, start, end, appIds)
}, statutils.GetFuncName(statMethod))
if err != nil {
log.Error("channel deduction statMethod failed!", log.Any("statItem", item), log.E(err))
return err
}
// 3.更新记录
if err := statrecordmod.UpsertItemRecord(ctx, statrecordmod.ChannelDeduction, item, end); err != nil {
log.Error("channel deduction commit UpsertItemRecord failed!", log.E(err))
return err
}
return nil
}