forked from all_user/stat_center
Initial commit
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
package statlandpage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"stat/common/log"
|
||||
"stat/common/statutils"
|
||||
"stat/common/utils"
|
||||
statrecordmod "stat/models/statrecord"
|
||||
"time"
|
||||
)
|
||||
|
||||
type StatMethod func(ctx context.Context, start, end time.Time) error
|
||||
type TimeAlignFunc func(t time.Time) time.Time
|
||||
|
||||
func timeAlignForDaily(t time.Time) time.Time {
|
||||
return utils.BeginWithTime(t)
|
||||
}
|
||||
func timeAlignForHour(t time.Time) time.Time {
|
||||
return utils.RecentMinute(t, 60)
|
||||
}
|
||||
|
||||
func RunStatHour(defaultStatDate time.Time) error {
|
||||
ctx := context.Background()
|
||||
job := statrecordmod.LandPageHourStatJob
|
||||
recordMap := defaultTimeMap(defaultStatDate)
|
||||
return recordStat(ctx, recordMap, job, time.Hour, -time.Hour, statLandPageHourApps, timeAlignForHour)
|
||||
}
|
||||
|
||||
func RunStatDaily(defaultStatDate time.Time) error {
|
||||
ctx := context.Background()
|
||||
job := statrecordmod.LandPageDailyStatJob
|
||||
recordMap := defaultTimeMap(defaultStatDate)
|
||||
return recordStat(ctx, recordMap, job, 24*time.Hour, -time.Hour, statLandPageDailyApps, timeAlignForDaily)
|
||||
}
|
||||
|
||||
// recordStat 统计
|
||||
func recordStat(ctx context.Context, recordMap map[string]time.Time, job string, timeStep, timeBack time.Duration, stat StatMethod, align TimeAlignFunc) error {
|
||||
recordResult, err := statrecordmod.LastRecordTimeMap(ctx, job) //第一次从默认记录时间开始统计
|
||||
if err != nil {
|
||||
log.Error("LastRecordTimeMap", log.E(err))
|
||||
return err
|
||||
}
|
||||
for _, record := range recordResult {
|
||||
recordMap[record.Item] = record.RecordTime
|
||||
}
|
||||
|
||||
now := time.Now().Add(timeBack)
|
||||
curStatTime := align(now)
|
||||
|
||||
for item, lastStatAt := range recordMap {
|
||||
_lastStatAt := align(lastStatAt)
|
||||
|
||||
for curStatTime.After(_lastStatAt) {
|
||||
end := _lastStatAt.Add(timeStep)
|
||||
|
||||
fmt.Printf("item:%s, start:%s, end:%s\n", item, _lastStatAt, end)
|
||||
|
||||
err := runStatMethod(ctx, item, job, _lastStatAt, end, stat)
|
||||
if err != nil {
|
||||
log.Error("recordStat failed!", log.E(err), log.Any("item", item), log.Any("job", job))
|
||||
return err
|
||||
}
|
||||
|
||||
_lastStatAt = end
|
||||
}
|
||||
|
||||
recordMap[item] = _lastStatAt
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// runStatMethod 调用统计项方法
|
||||
func runStatMethod(ctx context.Context, item, job string, start, end time.Time, stat StatMethod) error {
|
||||
var err error
|
||||
// 调用统计方法，获取数据
|
||||
err = statutils.ServerMon.RunWithName(func() error {
|
||||
return stat(ctx, start, end)
|
||||
}, statutils.GetFuncName(stat))
|
||||
|
||||
if err != nil {
|
||||
log.Error("runStatMethod failed!", log.Any("statItem", item), log.E(err))
|
||||
return err
|
||||
}
|
||||
|
||||
if err := statrecordmod.UpsertItemRecord(ctx, job, item, end); err != nil {
|
||||
log.Error("commit UpsertItemRecord failed!", log.E(err))
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package statlandpage
|
||||
|
||||
import (
|
||||
"stat/common/utils"
|
||||
"stat/tool/pubtest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRunStat(t *testing.T) {
|
||||
pubtest.TestInit("/Users/mac/Works/codes/stat_center/run/zhihu-test")
|
||||
s, _ := time.Parse(time.RFC3339, "2023-06-15T15:00:00+08:00")
|
||||
//curStatTime := utils.RecentMinute(s, 60) // 按60分钟对齐
|
||||
curStatTime := utils.BeginWithTime(s) // 按60分钟对齐
|
||||
//fmt.Printf("%s", curStatTime)
|
||||
|
||||
RunStatDaily(curStatTime)
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package statlandpage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"stat/common/log"
|
||||
"stat/common/utils"
|
||||
"stat/models/appInfoMod"
|
||||
"stat/models/downloadCpc"
|
||||
"stat/models/downloadinfo"
|
||||
"stat/models/downloadstat"
|
||||
"stat/models/registermod"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
const (
|
||||
LandPageStat string = "landPageStat" // 落地页统计
|
||||
)
|
||||
|
||||
func defaultTimeMap(defaultStatDate time.Time) map[string]time.Time {
|
||||
return map[string]time.Time{
|
||||
LandPageStat: defaultStatDate, // 落地页统计
|
||||
}
|
||||
}
|
||||
|
||||
func statLandPageDailyApps(ctx context.Context, start, end time.Time) error {
|
||||
appIds, err := appInfoMod.GetAppIds(ctx)
|
||||
if err != nil {
|
||||
log.Error("GetAppIds failed!", log.E(err))
|
||||
return err
|
||||
}
|
||||
for _, id := range appIds {
|
||||
if err := statLandPage(ctx, start, end, id, downloadstat.StatTypeDaily); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
time.Sleep(3 * time.Second)
|
||||
return nil
|
||||
}
|
||||
|
||||
func statLandPageHourApps(ctx context.Context, start, end time.Time) error {
|
||||
appIds, err := appInfoMod.GetAppIds(ctx)
|
||||
if err != nil {
|
||||
log.Error("GetAppIds failed!", log.E(err))
|
||||
return err
|
||||
}
|
||||
for _, id := range appIds {
|
||||
if err := statLandPage(ctx, start, end, id, downloadstat.StatTypeHour); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func statLandPage(ctx context.Context, start, end time.Time, appId int32, statType downloadstat.StatType) error {
|
||||
old, err := downloadstat.FindOneStat(ctx, bson.M{
|
||||
"appId": appId,
|
||||
"type": statType,
|
||||
"sumDate": start,
|
||||
})
|
||||
if err != nil {
|
||||
if err != mongo.ErrNoDocuments {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if old != nil {
|
||||
return nil // 已经存在
|
||||
}
|
||||
visitTotal := int64(0)
|
||||
visitDict := make(map[string]int64)
|
||||
visitList, err := downloadCpc.FindManyDownloadCpc(ctx, bson.M{
|
||||
"appId": appId,
|
||||
"createdAt": bson.M{"$gte": start, "$lt": end},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, visit := range visitList {
|
||||
visitTotal++
|
||||
visitDict[visit.IP]++
|
||||
}
|
||||
downTotal := int64(0)
|
||||
downDict := make(map[string]int64)
|
||||
downDictEx := make(map[string]int64)
|
||||
downList, err := downloadinfo.FindManyDownloadInfo(ctx, bson.M{
|
||||
"appId": appId,
|
||||
"createdAt": bson.M{"$gte": start.Add(-time.Hour), "$lt": end},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, down := range downList {
|
||||
downDictEx[down.IP]++
|
||||
if !down.CreatedAt.Before(start) && down.CreatedAt.Before(end) {
|
||||
downTotal++
|
||||
downDict[down.IP]++
|
||||
}
|
||||
}
|
||||
registerDict := make(map[string]int64)
|
||||
var registerCount int64
|
||||
err = registermod.RegisterUsersInfoByTimeShard(ctx, appId, start, end, utils.GlobalShardHour, func(item *registermod.RegisterMod) {
|
||||
if _, exists := downDictEx[item.IP]; exists {
|
||||
registerDict[item.IP]++
|
||||
registerCount++ // 这里只计算来自落地页的注册
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stat := &downloadstat.Stat{
|
||||
ID: primitive.NewObjectID(),
|
||||
AppId: appId,
|
||||
SumDate: start,
|
||||
EndDate: end,
|
||||
Hour: start.Hour(),
|
||||
Type: statType,
|
||||
VisitTotal: visitTotal,
|
||||
VisitUnique: int64(len(visitDict)),
|
||||
DownTotal: downTotal,
|
||||
DownUnique: int64(len(downDict)),
|
||||
InstallTotal: registerCount, // 落地页注册
|
||||
InstallUnique: int64(len(registerDict)),
|
||||
UpdatedAt: time.Now(),
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
|
||||
return downloadstat.InsertOneStat(ctx, stat)
|
||||
}
|
||||
Reference in New Issue
Block a user