Initial commit
This commit is contained in:
@@ -0,0 +1,260 @@
|
||||
package stathour
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"stat/common/enum/consts"
|
||||
"stat/common/log"
|
||||
"stat/common/statutils"
|
||||
"stat/common/utils"
|
||||
"stat/models/accessmod"
|
||||
"stat/models/appInfoMod"
|
||||
downloadinfo "stat/models/downloadinfo"
|
||||
"stat/models/minutestatmod"
|
||||
"stat/models/rechargemod"
|
||||
"stat/models/registermod"
|
||||
"stat/models/versionMod"
|
||||
"stat/statg"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
VisitCount string = "visitCount" // 访问数
|
||||
NewCount string = "newCount" // 新增数
|
||||
RechargeCount string = "rechargeCount" // 充值订单数
|
||||
LandpageClick string = "landpageClick" // 落地页访问次数
|
||||
|
||||
)
|
||||
|
||||
func defaultTimeMap(defaultStatDate time.Time) map[string]time.Time {
|
||||
return map[string]time.Time{
|
||||
RechargeCount: defaultStatDate, // 充值订单数
|
||||
NewCount: defaultStatDate, // 新增数
|
||||
VisitCount: defaultStatDate, // 访问数
|
||||
LandpageClick: defaultStatDate, // 落地页访问次数
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// 获取统计方法
|
||||
type StatMethod = func(ctx context.Context, start time.Time, end time.Time, appIds []int32) error
|
||||
|
||||
// 统计方法汇总
|
||||
var statMethodMap = map[string]StatMethod{
|
||||
RechargeCount: RechargeCountMethod, // 充值订单数
|
||||
NewCount: NewCountMethod, // 新增数
|
||||
VisitCount: VisitCountMethod, // 访问数
|
||||
LandpageClick: LandpageClickMethod, // 落地页访问次数
|
||||
}
|
||||
|
||||
// RechargeCountMethod 充值订单数
|
||||
func RechargeCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
|
||||
for _, appid := range appIds {
|
||||
// 获取充值金额
|
||||
itemList, err := rechargemod.TotalRechargeListV1(ctx, appid, start, end)
|
||||
if err != nil {
|
||||
log.Error("DirectAmountMethod DirectRechargeAmount ", log.E(err))
|
||||
return err
|
||||
}
|
||||
stat := &statutils.Stat{}
|
||||
for _, item := range itemList {
|
||||
client := statutils.GetClientInfo(item.SysType, item.DevType, item.DistrictCode, item.IsDirect, false, item.IsFormalUser)
|
||||
stat.Add(item.PayMoney, client)
|
||||
}
|
||||
// 更新数据库
|
||||
doc := &minutestatmod.MinuteStatDoc{
|
||||
RechargetCount: stat.CountRef(),
|
||||
RechargetMoney: stat.ValueRef(),
|
||||
DirectAmount: stat.Total.Direct.ValueRef(),
|
||||
FissionAmount: &stat.Total.Fission.Value,
|
||||
DistrictRecharge: &stat.Total.District.Value,
|
||||
WebRecharge: &stat.Total.Web.Value,
|
||||
NormalRecharge: &stat.Total.Normal.Value,
|
||||
AndroidRecharge: stat.Android.ValueRef(),
|
||||
IosRecharge: stat.Ios.ValueRef(),
|
||||
LightRecharge: stat.Light.ValueRef(),
|
||||
H5Recharge: stat.H5.ValueRef(),
|
||||
}
|
||||
err = minutestatmod.UpsertMinuteDocStat(ctx, start, appid, doc)
|
||||
if err != nil {
|
||||
log.Error("RechargeCountMethod UpsertDailyStat ", log.E(err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewCountMethod 新增数
|
||||
func NewCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
|
||||
for _, appid := range appIds {
|
||||
stat := &statutils.Stat{}
|
||||
userDict := make(map[uint64]bool)
|
||||
err := registermod.RegisterFormalsInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
|
||||
if item.IsFormal && statutils.IsClientH5OrLight(item.SysType, item.DevType) {
|
||||
if _, exists := userDict[item.UserId]; !exists {
|
||||
userDict[item.UserId] = true
|
||||
client := statutils.GetClientInfo(item.SysType, item.DevType,
|
||||
item.DistrictCode, item.IsDirect, false, item.IsFormal)
|
||||
stat.Add(1, client)
|
||||
}
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = registermod.RegisterUsersInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
|
||||
userDict[item.UserId] = true
|
||||
client := statutils.GetClientInfo(item.SysType, item.DevType,
|
||||
item.DistrictCode, item.IsDirect, false, item.IsFormal)
|
||||
stat.Add(1, client)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 更新数据库
|
||||
doc := &minutestatmod.MinuteStatDoc{
|
||||
NewCount: stat.Total.CountRef(),
|
||||
DirectAddCount: stat.Total.Direct.CountRef(),
|
||||
FissionAddCount: &stat.Total.Fission.Count,
|
||||
WebAddCount: &stat.Total.Web.Count,
|
||||
DistrictAddCount: &stat.Total.District.Count,
|
||||
NormalAddCount: &stat.Total.Normal.Count,
|
||||
AndroidAddCount: stat.Android.CountRef(),
|
||||
IosAddCount: stat.Ios.CountRef(),
|
||||
LightGuestAddCount: stat.Light.Guest.CountRef(),
|
||||
LightFormalAddCount: stat.Light.Formal.CountRef(),
|
||||
H5AddCount: stat.H5.CountRef(),
|
||||
H5FormalAddCount: stat.H5.Formal.CountRef(),
|
||||
}
|
||||
err = minutestatmod.UpsertMinuteDocStat(ctx, start, appid, doc)
|
||||
if err != nil {
|
||||
log.Error("DirectVisitCountMethod UpsertDailyStat ", log.E(err))
|
||||
return err
|
||||
}
|
||||
|
||||
iosCount := *stat.Ios.CountRef()
|
||||
androidCount := *stat.Android.CountRef()
|
||||
|
||||
if statg.Conf.RunMod == consts.RunModRelease && appid == 888 {
|
||||
if iosCount < 5 || androidCount <= 40 {
|
||||
appinfo, _ := appInfoMod.GetAppInfoById(ctx, appid)
|
||||
text := fmt.Sprintf("[警告]:应用名称:%s\n10分钟新增异常:\n时间:%s\nandroid:%d\nios:%d\n", appinfo.Name, start, androidCount, iosCount)
|
||||
err = statg.Bot.SendMonitorMessage(text)
|
||||
if err != nil {
|
||||
log.Error("tg Send ", log.E(err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if statg.Conf.RunMod == consts.RunModRelease && appid == 999 {
|
||||
if iosCount < 5 || androidCount <= 40 {
|
||||
appinfo, _ := appInfoMod.GetAppInfoById(ctx, appid)
|
||||
text := fmt.Sprintf("[警告]:应用名称:%s\n10分钟新增异常:\n时间:%s\nandroid:%d\nios:%d\n", appinfo.Name, start, androidCount, iosCount)
|
||||
err = statg.Bot.SendMonitorMessage(text)
|
||||
if err != nil {
|
||||
log.Error("tg Send ", log.E(err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if statg.Conf.RunMod == consts.RunModRelease && (appid == 222 || appid == 80) {
|
||||
if iosCount <= 2 || androidCount <= 10 {
|
||||
appinfo, _ := appInfoMod.GetAppInfoById(ctx, appid)
|
||||
text := fmt.Sprintf("[警告]:应用名称:%s\n10分钟新增异常:\n时间:%s\nandroid:%d\nios:%d\n", appinfo.Name, start, androidCount, iosCount)
|
||||
err = statg.Bot.SendMonitorMessage(text)
|
||||
if err != nil {
|
||||
log.Error("tg Send ", log.E(err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// VisitCountMethod 访问数
|
||||
func VisitCountMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
|
||||
start = start.Add(-time.Minute * 10)
|
||||
end = end.Add(-time.Minute * 10)
|
||||
for _, appid := range appIds {
|
||||
stat := &statutils.Stat{}
|
||||
err := accessmod.VisitInfoByTimeShard(ctx, appid, start, end, utils.GlobalShardHour, func(item *accessmod.AccessMod) {
|
||||
client := statutils.GetClientInfo(item.SysType, item.DevType,
|
||||
item.DistrictCode, item.IsDirect, false, item.IsFormalUser)
|
||||
stat.Add(1, client)
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("AppVisitCountMethod:VisitInfoByTimeShard", log.E(err))
|
||||
return err
|
||||
}
|
||||
// 更新数据库
|
||||
doc := &minutestatmod.MinuteStatDoc{
|
||||
AccessCount: stat.CountRef(),
|
||||
DirectAccessCount: stat.Total.Direct.CountRef(),
|
||||
NaturalAccessCount: stat.Total.Direct.Natural.CountRef(),
|
||||
FissionAccessCount: stat.Total.Fission.CountRef(),
|
||||
DistrictAccessCount: stat.Total.District.CountRef(),
|
||||
WebAccessCount: stat.Total.Web.CountRef(),
|
||||
NormalAccessCount: stat.Total.Normal.CountRef(),
|
||||
AndroidAccessCount: stat.Android.CountRef(),
|
||||
IosAccessCount: stat.Ios.CountRef(),
|
||||
H5AccessCount: stat.H5.CountRef(),
|
||||
LightAccessCount: stat.Light.CountRef(),
|
||||
}
|
||||
err = minutestatmod.UpsertMinuteDocStat(ctx, start, appid, doc)
|
||||
if err != nil {
|
||||
log.Error("DirectVisitCountMethod UpsertDailyStat ", log.E(err))
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LandpageClickMethod 落地页访问次数
|
||||
func LandpageClickMethod(ctx context.Context, start time.Time, end time.Time, appIds []int32) error {
|
||||
for _, appid := range appIds {
|
||||
|
||||
downloadList, err := downloadinfo.GetAccessCountByTime(ctx, appid, start, end)
|
||||
if err != nil {
|
||||
log.Error("RunMonitor, GetAccessCountByTime", log.E(err))
|
||||
return err
|
||||
}
|
||||
androidClick := int64(0)
|
||||
appstoreClick := int64(0)
|
||||
iosAtherClick := int64(0)
|
||||
h5Click := int64(0)
|
||||
|
||||
for _, downloadInfo := range downloadList {
|
||||
if downloadInfo.SysType == "android" {
|
||||
androidClick += 1
|
||||
} else if strings.Contains(downloadInfo.SysType, "ios") || strings.Contains(downloadInfo.SysType, "light") {
|
||||
if downloadInfo.PackageType == versionMod.Shop {
|
||||
appstoreClick += 1
|
||||
} else if downloadInfo.PackageType == versionMod.Enterprise {
|
||||
iosAtherClick += 1
|
||||
} else if downloadInfo.PackageType == versionMod.Light {
|
||||
h5Click += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
// 更新数据库
|
||||
doc := &minutestatmod.MinuteStatDoc{
|
||||
LandpageAndroidClick: &androidClick,
|
||||
LandpageIosAppstoreClick: &appstoreClick,
|
||||
LandpageIosAthorClick: &iosAtherClick,
|
||||
LandpageIosH5Click: &h5Click,
|
||||
}
|
||||
err = minutestatmod.UpsertMinuteDocStat(ctx, start, appid, doc)
|
||||
if err != nil {
|
||||
log.Error("DirectVisitCountMethod UpsertDailyStat ", log.E(err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package stathour
|
||||
|
||||
import (
|
||||
"context"
|
||||
"stat/common/log"
|
||||
"stat/common/statutils"
|
||||
"stat/common/utils"
|
||||
"stat/models/appInfoMod"
|
||||
statrecordmod "stat/models/statrecord"
|
||||
"time"
|
||||
)
|
||||
|
||||
/*
|
||||
* 主要统计任务:
|
||||
* 按1个小时一个点，记录实时数据 app（新增、访问数....）
|
||||
*/
|
||||
|
||||
// RunStat // 按10分钟的频率去统计
|
||||
func RunStat(defaultStatDate time.Time) error {
|
||||
log.Info("APP hour stat ......")
|
||||
ctx := context.Background()
|
||||
recordMap := defaultTimeMap(defaultStatDate)
|
||||
|
||||
// 1.获取统计项上次统计的时候(每天执行)
|
||||
recoredResult, err := statrecordmod.LastRecordTimeMap(ctx, statrecordmod.AppHour) //第一次从默认记录时间开始统计
|
||||
if err != nil {
|
||||
log.Error("APP hour stat ", log.E(err))
|
||||
return err
|
||||
}
|
||||
for _, record := range recoredResult {
|
||||
recordMap[record.Item] = record.RecordTime
|
||||
}
|
||||
|
||||
now := time.Now().Add(time.Minute * -5) // 延迟5分钟，数据同步完成
|
||||
statFrq := 10 * time.Minute // 统计频率
|
||||
curStatTime := utils.RecentMinute(now, 10) // 按10分钟对齐
|
||||
|
||||
appIds, err := appInfoMod.GetAppIds(ctx)
|
||||
if err != nil {
|
||||
log.Error("APP hour stat GetAppIds failed!", log.E(err))
|
||||
return err
|
||||
}
|
||||
|
||||
for item, lastStatAt := range recordMap {
|
||||
_lastStatAt := lastStatAt
|
||||
|
||||
for curStatTime.After(_lastStatAt) {
|
||||
end := _lastStatAt.Add(statFrq)
|
||||
// 超过本次统计结束时间，则结束
|
||||
if end.After(curStatTime) {
|
||||
end = curStatTime
|
||||
}
|
||||
// 根据时间段统计数据
|
||||
//fmt.Println("APP hour stat ", _lastStatAt, end)
|
||||
err := runStatMethod(ctx, item, _lastStatAt, end, appIds)
|
||||
if err != nil {
|
||||
log.Error("APP hour stat runStatMethod failed!", log.E(err), log.Any("item", item))
|
||||
return err
|
||||
}
|
||||
|
||||
_lastStatAt = end
|
||||
}
|
||||
recordMap[item] = _lastStatAt
|
||||
}
|
||||
log.Info("APP hour stat ")
|
||||
return nil
|
||||
}
|
||||
|
||||
// runStatMethod 调用统计项方法
|
||||
func runStatMethod(ctx context.Context, item string, start, end time.Time, appIds []int32) (err error) {
|
||||
// 1.获取统计方法
|
||||
statMethod, ok := statMethodMap[item]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 调用统计方法，获取数据
|
||||
err = statutils.ServerMon.RunWithName(func() error {
|
||||
return statMethod(ctx, start, end, appIds)
|
||||
}, statutils.GetFuncName(statMethod))
|
||||
|
||||
if err != nil {
|
||||
log.Error("APP hour stat statMethod failed!", log.Any("statItem", item), log.E(err))
|
||||
return err
|
||||
}
|
||||
|
||||
// 3.更新记录
|
||||
if err := statrecordmod.UpsertItemRecord(ctx, statrecordmod.AppHour, item, end); err != nil {
|
||||
log.Error("APP hour stat commit UpsertItemRecord failed!", log.E(err))
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user