package monitor import ( "bytes" "fmt" "io/ioutil" "math" "net/http" "runtime" "sort" "time" "videoweb/pub/common/enum/consts" "videoweb/pub/common/enum/redisconst" "videoweb/pub/common/godef" "videoweb/pub/common/ipaddr" "videoweb/pub/common/log" "videoweb/pub/common/srvdef" "videoweb/pub/common/tgbot" "videoweb/pub/redis" "github.com/gin-gonic/gin" ) // Check 运行监控检查 func Check(ctx *gin.Context) { //查看是否有限制登陆 ip := ipaddr.ResolveClientIP(ctx) ifKey := redisconst.IpForbitKey(ip) retValue, _ := redis.Get(ifKey) if retValue != nil { //log.Info("Test Check", log.Any("ip", ip), log.Any("retValue", *retValue)) ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ "code": srvdef.ErrIpForbid, "msg": srvdef.ErrIpForbid.Msg(), }) return } // Start timer start := time.Now() path := ctx.Request.URL.Path raw := ctx.Request.URL.RawQuery rawData, err := ctx.GetRawData() if err != nil { log.Info("Test get raw data error", log.E(err)) } // 将取出来的执行条件放回ctx中 ctx.Request.Body = ioutil.NopCloser(bytes.NewBuffer(rawData)) //recover func() { defer func() { if r := recover(); r != nil { buf := make([]byte, 2048) length := runtime.Stack(buf, false) info := string(buf[:length]) log.Error("catch a unkown excetion", log.Any("excetion :", info)) //发送tg消息 msg := "monitor check crash,the stack:\n" + info tgbot.SendToProgram("API接口处理宕了", msg) } }() // Process request ctx.Next() }() // Stop timer stopTime := time.Now() latency := stopTime.Sub(start).Milliseconds() if raw != "" { path = path + "?" + raw } //添加记录 //log.Info("Test", log.Any("rawData", rawData)) rawString := string(rawData) if path == "/api/app/feedback/add" { rawString = "" } addApiCall(path, int64(latency), rawString) //如果时间大于1秒发送tg消息 if latency > 4000 { userId := ctx.GetInt64(consts.CtxUserID) msg := "" msg = msg + fmt.Sprintf("path:%s", path) + "\n" msg = msg + fmt.Sprintf("userId:%d", userId) + "\n" msg = msg + fmt.Sprintf("userIp:%s", ip) + "\n" msg = msg + fmt.Sprintf("rawData:%s", string(rawData)) + "\n" msg = msg + fmt.Sprintf("time:%d", latency) + "\n" tgbot.SendToProgram("API接口调用超时", msg) } //log.Info("monitor:Check", log.Any("latency", latency), log.Any("path", path)) } func init() { hourStatises = make(map[string]*LogicStatis) muniteStatises = make(map[string]*LogicStatis) apiCallChan = make(chan *StatisParam, 1024) godef.Go(loop) log.Info("(monitor:init) init success.") } func loop() { log.Info("(monitor:loop) start...") defer log.Info("monitor:loop) end") var param *StatisParam for { select { case param = <-apiCallChan: onStatis(param) default: time.Sleep(time.Millisecond * 10) } } } func onStatis(param *StatisParam) { if param == nil { return } now := time.Now() hourStr := now.Format("2006-01-02 15") hourStatis, hourOk := hourStatises[hourStr] if !hourOk { hourStatis = &LogicStatis{ ApisStatis: make(map[string]*APIStatis), } hourStatises[hourStr] = hourStatis } muniteStr := now.Format("2006-01-02 15:04") minuteStatis, minuteOk := muniteStatises[muniteStr] if !minuteOk { minuteStatis = &LogicStatis{ ApisStatis: make(map[string]*APIStatis), } muniteStatises[muniteStr] = minuteStatis } onStatisApiCall(hourStr, param, hourStatis, 10000000) onStatisApiCall(muniteStr, param, minuteStatis, 20000) //如果新的一小时,发送上一小时数据 if !hourOk { //清空分钟数据 muniteStatises = make(map[string]*LogicStatis) lastHour := now.Add(-time.Hour).Format("2006-01-02 15") hourStatis, hourOk = hourStatises[lastHour] if hourOk { msg := fmt.Sprintf("统计时间:%s\n", lastHour) msg += formatStatis(hourStatis) tgbot.SendToProgram("API接口调用统计", msg) } } } func formatStatis(hourStatis *LogicStatis) string { apiStatises := make([]*APIStatis, 0) for _, static := range hourStatis.ApisStatis { apiStatises = append(apiStatises, static) } sort.Slice(apiStatises, func(i, j int) bool { return apiStatises[i].CallNum > apiStatises[j].CallNum }) //发送数据 msg := "" maxlen := len(apiStatises) if maxlen > 10 { maxlen = 10 } for i := 0; i < maxlen; i++ { apiStatis := apiStatises[i] msg += formatApiStatis(apiStatis) } timePerCall := int64(0) callPerSec := int64(0) if hourStatis.ApiCallNum > 0 { timePerCall = hourStatis.ApiTotalTime / hourStatis.ApiCallNum diffSecdonds := int64(time.Now().Sub(hourStatis.Start).Seconds()) if diffSecdonds > 0 { callPerSec = hourStatis.ApiCallNum / diffSecdonds } } msg += fmt.Sprintf("ApiCallNum:%d \n", hourStatis.ApiCallNum) msg += fmt.Sprintf("ApiTotalTime:%d \n", hourStatis.ApiTotalTime) msg += fmt.Sprintf("ApiTimePerCall:%d \n", timePerCall) msg += fmt.Sprintf("ApiCallPerSec:%d \n", callPerSec) return msg } func formatApiStatis(apiStatis *APIStatis) string { msg := "" if apiStatis.CallNum > 0 { apiStatis.TimePerCall = apiStatis.TotalTime / apiStatis.CallNum diffSeconds := int64(time.Now().Sub(apiStatis.Start).Seconds()) if diffSeconds > 0 { apiStatis.CallPerSec = apiStatis.CallNum / diffSeconds } } statisStr := fmt.Sprintf(`path %s: MaxRawData:%s CallNum:%d;MinTime:%d;MaxTime:%d;TotalTime:%d;TimePerCall:%d;CallPerSec:%d`, apiStatis.Path, apiStatis.MaxTimeRawData, apiStatis.CallNum, apiStatis.MinTime, apiStatis.MaxTime, apiStatis.TotalTime, apiStatis.TimePerCall, apiStatis.CallPerSec) msg += statisStr msg += "\n---------------------------------------------------\n" return msg } // api统计 func onStatisApiCall(dateStr string, param *StatisParam, logicStatis *LogicStatis, maxCallNum int64) { logicStatis.ApiCallNum++ logicStatis.ApiTotalTime += param.Param2 apiStatis, ok := logicStatis.ApisStatis[param.Param1] if !ok { apiStatis = &APIStatis{ Path: param.Param1, MaxTimeRawData: param.Param3, CallNum: 0, MinTime: math.MaxInt64, MaxTime: 0, TotalTime: 0, Start: time.Now(), Send: 0, } logicStatis.ApisStatis[param.Param1] = apiStatis } apiStatis.CallNum++ apiStatis.TotalTime += param.Param2 if param.Param2 > apiStatis.MaxTime { apiStatis.MaxTime = param.Param2 apiStatis.MaxTimeRawData = param.Param3 } if param.Param2 < apiStatis.MinTime { apiStatis.MinTime = param.Param2 } if apiStatis.CallNum > maxCallNum && apiStatis.Send < 4 { apiStatis.Send++ msg := fmt.Sprintf("接口%s每分钟调用超过最大次数%d\n", apiStatis.Path, maxCallNum) msg += formatApiStatis(apiStatis) tgbot.SendToProgram("API调用警告", msg) } } func addApiCall(path string, latency int64, rawData string) { param := &StatisParam{ Type: SP_ApiCall, Param1: path, Param2: latency, Param3: rawData, } pushParam(param) } func pushParam(param *StatisParam) { //no block select { case apiCallChan <- param: default: info := "(monitor:pushParam) failed,chan is full." log.Info(info) tgbot.SendToProgram("monitor pushParam failed", info) } }