263 lines
6.9 KiB
Go
263 lines
6.9 KiB
Go
package monitor
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"math"
|
|
"net/http"
|
|
"runtime"
|
|
"sort"
|
|
"time"
|
|
"videoweb/pub/common/enum/consts"
|
|
"videoweb/pub/common/enum/redisconst"
|
|
"videoweb/pub/common/godef"
|
|
"videoweb/pub/common/log"
|
|
"videoweb/pub/common/srvdef"
|
|
"videoweb/pub/common/tgbot"
|
|
"videoweb/pub/redis"
|
|
|
|
"github.com/gin-gonic/gin"
|
|
)
|
|
|
|
// Check 运行监控检查
|
|
func Check(ctx *gin.Context) {
|
|
//查看是否有限制登陆
|
|
ip := ctx.ClientIP()
|
|
ifKey := redisconst.IpForbitKey(ip)
|
|
retValue, _ := redis.Get(ifKey)
|
|
if retValue != nil {
|
|
//log.Info("Test Check", log.Any("ip", ip), log.Any("retValue", *retValue))
|
|
ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
|
|
"code": srvdef.ErrIpForbid,
|
|
"msg": srvdef.ErrIpForbid.Msg(),
|
|
})
|
|
return
|
|
}
|
|
|
|
// Start timer
|
|
start := time.Now()
|
|
path := ctx.Request.URL.Path
|
|
raw := ctx.Request.URL.RawQuery
|
|
rawData, err := ctx.GetRawData()
|
|
if err != nil {
|
|
log.Info("Test get raw data error", log.E(err))
|
|
}
|
|
// 将取出来的执行条件放回ctx中
|
|
ctx.Request.Body = ioutil.NopCloser(bytes.NewBuffer(rawData))
|
|
//recover
|
|
func() {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
buf := make([]byte, 2048)
|
|
length := runtime.Stack(buf, false)
|
|
info := string(buf[:length])
|
|
log.Error("catch a unkown excetion", log.Any("excetion :", info))
|
|
//发送tg消息
|
|
msg := "monitor check crash,the stack:\n" + info
|
|
tgbot.SendToProgram("API接口处理宕了", msg)
|
|
}
|
|
}()
|
|
// Process request
|
|
ctx.Next()
|
|
}()
|
|
// Stop timer
|
|
stopTime := time.Now()
|
|
latency := stopTime.Sub(start).Milliseconds()
|
|
if raw != "" {
|
|
path = path + "?" + raw
|
|
}
|
|
//添加记录
|
|
//log.Info("Test", log.Any("rawData", rawData))
|
|
rawString := string(rawData)
|
|
addApiCall(path, int64(latency), rawString)
|
|
//如果时间大于1秒发送tg消息
|
|
if latency > 4000 {
|
|
userId := ctx.GetInt64(consts.CtxUserID)
|
|
msg := ""
|
|
msg = msg + fmt.Sprintf("path:%s", path) + "\n"
|
|
msg = msg + fmt.Sprintf("userId:%d", userId) + "\n"
|
|
msg = msg + fmt.Sprintf("userIp:%s", ip) + "\n"
|
|
msg = msg + fmt.Sprintf("rawData:%s", string(rawData)) + "\n"
|
|
msg = msg + fmt.Sprintf("time:%d", latency) + "\n"
|
|
tgbot.SendToProgram("API接口调用超时", msg)
|
|
}
|
|
//log.Info("monitor:Check", log.Any("latency", latency), log.Any("path", path))
|
|
}
|
|
|
|
func init() {
|
|
hourStatises = make(map[string]*LogicStatis)
|
|
muniteStatises = make(map[string]*LogicStatis)
|
|
apiCallChan = make(chan *StatisParam, 1024)
|
|
godef.Go(loop)
|
|
log.Info("(monitor:init) init success.")
|
|
}
|
|
|
|
func loop() {
|
|
log.Info("(monitor:loop) start...")
|
|
defer log.Info("monitor:loop) end")
|
|
var param *StatisParam
|
|
for {
|
|
select {
|
|
case param = <-apiCallChan:
|
|
onStatis(param)
|
|
default:
|
|
time.Sleep(time.Millisecond * 10)
|
|
}
|
|
}
|
|
}
|
|
|
|
func onStatis(param *StatisParam) {
|
|
if param == nil {
|
|
return
|
|
}
|
|
now := time.Now()
|
|
hourStr := now.Format("2006-01-02 15")
|
|
hourStatis, hourOk := hourStatises[hourStr]
|
|
if !hourOk {
|
|
hourStatis = &LogicStatis{
|
|
ApisStatis: make(map[string]*APIStatis),
|
|
}
|
|
hourStatises[hourStr] = hourStatis
|
|
}
|
|
muniteStr := now.Format("2006-01-02 15:04")
|
|
minuteStatis, minuteOk := muniteStatises[muniteStr]
|
|
if !minuteOk {
|
|
minuteStatis = &LogicStatis{
|
|
ApisStatis: make(map[string]*APIStatis),
|
|
}
|
|
muniteStatises[muniteStr] = minuteStatis
|
|
}
|
|
onStatisApiCall(hourStr, param, hourStatis, 10000000)
|
|
onStatisApiCall(muniteStr, param, minuteStatis, 20000)
|
|
//如果新的一小时,发送上一小时数据
|
|
if !hourOk {
|
|
//清空分钟数据
|
|
muniteStatises = make(map[string]*LogicStatis)
|
|
lastHour := now.Add(-time.Hour).Format("2006-01-02 15")
|
|
hourStatis, hourOk = hourStatises[lastHour]
|
|
if hourOk {
|
|
msg := fmt.Sprintf("统计时间:%s\n", lastHour)
|
|
msg += formatStatis(hourStatis)
|
|
tgbot.SendToProgram("API接口调用统计", msg)
|
|
}
|
|
}
|
|
}
|
|
|
|
func formatStatis(hourStatis *LogicStatis) string {
|
|
apiStatises := make([]*APIStatis, 0)
|
|
for _, static := range hourStatis.ApisStatis {
|
|
apiStatises = append(apiStatises, static)
|
|
}
|
|
sort.Slice(apiStatises, func(i, j int) bool {
|
|
return apiStatises[i].CallNum > apiStatises[j].CallNum
|
|
})
|
|
//发送数据
|
|
msg := ""
|
|
maxlen := len(apiStatises)
|
|
if maxlen > 10 {
|
|
maxlen = 10
|
|
}
|
|
for i := 0; i < maxlen; i++ {
|
|
apiStatis := apiStatises[i]
|
|
msg += formatApiStatis(apiStatis)
|
|
}
|
|
timePerCall := int64(0)
|
|
callPerSec := int64(0)
|
|
if hourStatis.ApiCallNum > 0 {
|
|
timePerCall = hourStatis.ApiTotalTime / hourStatis.ApiCallNum
|
|
diffSecdonds := int64(time.Now().Sub(hourStatis.Start).Seconds())
|
|
if diffSecdonds > 0 {
|
|
callPerSec = hourStatis.ApiCallNum / diffSecdonds
|
|
}
|
|
}
|
|
msg += fmt.Sprintf("ApiCallNum:%d \n", hourStatis.ApiCallNum)
|
|
msg += fmt.Sprintf("ApiTotalTime:%d \n", hourStatis.ApiTotalTime)
|
|
msg += fmt.Sprintf("ApiTimePerCall:%d \n", timePerCall)
|
|
msg += fmt.Sprintf("ApiCallPerSec:%d \n", callPerSec)
|
|
return msg
|
|
}
|
|
|
|
func formatApiStatis(apiStatis *APIStatis) string {
|
|
msg := ""
|
|
if apiStatis.CallNum > 0 {
|
|
apiStatis.TimePerCall = apiStatis.TotalTime / apiStatis.CallNum
|
|
diffSeconds := int64(time.Now().Sub(apiStatis.Start).Seconds())
|
|
if diffSeconds > 0 {
|
|
apiStatis.CallPerSec = apiStatis.CallNum / diffSeconds
|
|
}
|
|
}
|
|
|
|
statisStr := fmt.Sprintf(`path %s:
|
|
MaxRawData:%s
|
|
CallNum:%d;MinTime:%d;MaxTime:%d;TotalTime:%d;TimePerCall:%d;CallPerSec:%d`,
|
|
apiStatis.Path,
|
|
apiStatis.MaxTimeRawData,
|
|
apiStatis.CallNum,
|
|
apiStatis.MinTime,
|
|
apiStatis.MaxTime,
|
|
apiStatis.TotalTime,
|
|
apiStatis.TimePerCall,
|
|
apiStatis.CallPerSec)
|
|
msg += statisStr
|
|
msg += "\n---------------------------------------------------\n"
|
|
return msg
|
|
}
|
|
|
|
// api统计
|
|
func onStatisApiCall(dateStr string, param *StatisParam,
|
|
logicStatis *LogicStatis, maxCallNum int64) {
|
|
logicStatis.ApiCallNum++
|
|
logicStatis.ApiTotalTime += param.Param2
|
|
apiStatis, ok := logicStatis.ApisStatis[param.Param1]
|
|
if !ok {
|
|
apiStatis = &APIStatis{
|
|
Path: param.Param1,
|
|
MaxTimeRawData: param.Param3,
|
|
CallNum: 0,
|
|
MinTime: math.MaxInt64,
|
|
MaxTime: 0,
|
|
TotalTime: 0,
|
|
Start: time.Now(),
|
|
Send: 0,
|
|
}
|
|
logicStatis.ApisStatis[param.Param1] = apiStatis
|
|
}
|
|
apiStatis.CallNum++
|
|
apiStatis.TotalTime += param.Param2
|
|
if param.Param2 > apiStatis.MaxTime {
|
|
apiStatis.MaxTime = param.Param2
|
|
apiStatis.MaxTimeRawData = param.Param3
|
|
}
|
|
if param.Param2 < apiStatis.MinTime {
|
|
apiStatis.MinTime = param.Param2
|
|
}
|
|
if apiStatis.CallNum > maxCallNum && apiStatis.Send < 4 {
|
|
apiStatis.Send++
|
|
msg := fmt.Sprintf("接口%s每分钟调用超过最大次数%d\n", apiStatis.Path, maxCallNum)
|
|
msg += formatApiStatis(apiStatis)
|
|
tgbot.SendToProgram("API调用警告", msg)
|
|
}
|
|
}
|
|
|
|
func addApiCall(path string, latency int64, rawData string) {
|
|
param := &StatisParam{
|
|
Type: SP_ApiCall,
|
|
Param1: path,
|
|
Param2: latency,
|
|
Param3: rawData,
|
|
}
|
|
pushParam(param)
|
|
}
|
|
|
|
func pushParam(param *StatisParam) {
|
|
//no block
|
|
select {
|
|
case apiCallChan <- param:
|
|
default:
|
|
info := "(monitor:pushParam) failed,chan is full."
|
|
log.Info(info)
|
|
tgbot.SendToProgram("monitor pushParam failed", info)
|
|
}
|
|
}
|