Initial commit: web_hghl_server
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
@@ -0,0 +1,140 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"videoweb/app/appg"
|
||||
"videoweb/pub/common/enum/consts"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/srvdef"
|
||||
"videoweb/pub/common/utils"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// gin 验证中间件
|
||||
|
||||
var (
|
||||
// 白名单，不需要验证token的api
|
||||
whiteList = []string{
|
||||
"/swagger",
|
||||
"/api/app/ping/check",
|
||||
"/api/app/ping/config",
|
||||
"/api/app/health/ping",
|
||||
"/api/app/login/guest",
|
||||
"/api/app/auth/register",
|
||||
"/api/app/auth/login",
|
||||
"/api/app/user/qrcode/info",
|
||||
"/api/app/upload/images",
|
||||
"/api/app/home",
|
||||
"/api/app/richtext",
|
||||
"/api/app/content/tag/list",
|
||||
"/api/app/config/footer/info",
|
||||
"/api/app/config/richtext/list",
|
||||
"/api/app/navigation/header",
|
||||
"/api/app/comment/list",
|
||||
"/api/app/video/more",
|
||||
"/api/app/video/list",
|
||||
"/api/app/video/category/list",
|
||||
"/api/app/video/info",
|
||||
"/api/app/video/recommend/list",
|
||||
"/api/app/video/guess/list",
|
||||
"/api/app/video/rank/list",
|
||||
"/api/app/playlet/recommend/list",
|
||||
"/api/app/playlet/guess/list",
|
||||
"/api/app/playlet/home",
|
||||
"/api/app/playlet/home-config/list",
|
||||
"/api/app/playlet/topic/detail",
|
||||
"/api/app/playlet/theme/list",
|
||||
"/api/app/playlet/list",
|
||||
"/api/app/playlet/content/list",
|
||||
"/api/app/playlet/tag/",
|
||||
"/api/app/playlet/info",
|
||||
"/api/app/playlet/chapter/info",
|
||||
"/api/app/playlet/rank/list",
|
||||
"/api/app/actress",
|
||||
"/api/app/upload/videoBody",
|
||||
"/api/app/upload/videoOk",
|
||||
"/api/app/upload/image",
|
||||
"/search",
|
||||
"/api/app/applist",
|
||||
"/h5/m3u8",
|
||||
"/enkey",
|
||||
"/newenkey",
|
||||
"/lsjenkey",
|
||||
"/dh",
|
||||
"/businessCooperation/",
|
||||
"/servicesProject/",
|
||||
"/consulting/",
|
||||
"/consultinginfo/",
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
tokener = utils.NewTokenRedis()
|
||||
)
|
||||
|
||||
func Auth(ctx *gin.Context) {
|
||||
dotoken := func() srvdef.CodeType {
|
||||
token := ctx.Request.Header.Get("Authorization")
|
||||
// 去掉Bearer，如果有
|
||||
token = utils.StripBearerPrefixFromTokenString(token)
|
||||
|
||||
if token == "" {
|
||||
for _, w := range whiteList {
|
||||
if strings.Contains(ctx.Request.URL.Path, w) {
|
||||
return srvdef.Success
|
||||
}
|
||||
}
|
||||
|
||||
log.Error("Auth:token is empty", log.Any("sysType", ctx.GetString(consts.CtxSysType)))
|
||||
return srvdef.InvalidToken
|
||||
}
|
||||
|
||||
userId, newToken, err := utils.ParseUserToken(token, appg.Conf.JwtKey)
|
||||
if err != nil {
|
||||
for _, w := range whiteList {
|
||||
if strings.Contains(ctx.Request.URL.Path, w) {
|
||||
return srvdef.Success
|
||||
}
|
||||
}
|
||||
|
||||
log.Error("Auth:ParseUserToken", log.Any("token", token), log.E(err))
|
||||
return srvdef.InvalidToken
|
||||
}
|
||||
|
||||
has, err := tokener.Exists(userId, token)
|
||||
if err != nil {
|
||||
log.Error("Auth:Exists", log.Any("userId", userId), log.Any("token", token), log.E(err))
|
||||
return srvdef.InvalidToken
|
||||
}
|
||||
if !has {
|
||||
log.Error("Auth:Exists:token not exists", log.Any("userId", userId), log.Any("token", token), log.E(err))
|
||||
return srvdef.InvalidToken
|
||||
}
|
||||
|
||||
if len(newToken) > 0 {
|
||||
err = tokener.Save(userId, newToken)
|
||||
if err != nil {
|
||||
log.Error("Auth:Save", log.Any("userId", userId), log.Any("token", newToken), log.E(err))
|
||||
return srvdef.StoreLevel
|
||||
}
|
||||
ctx.Writer.Header().Set("Refresh-Authorization", newToken)
|
||||
}
|
||||
|
||||
ctx.Set(consts.CtxUserID, userId)
|
||||
ctx.Set(consts.CtxToken, token)
|
||||
|
||||
return srvdef.Success
|
||||
}
|
||||
|
||||
errType := dotoken()
|
||||
if errType != srvdef.Success {
|
||||
// log.Info("Auth bad request", log.Any("Path", ctx.Request.URL.Path), log.Any("ip", ctx.ClientIP()))
|
||||
ctx.AbortWithStatusJSON(http.StatusOK, gin.H{
|
||||
"code": errType,
|
||||
"msg": errType.Msg(),
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package cors
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
Origin *[]string //传入空指针，表示允许"*", 传入空数组表示禁止跨域
|
||||
Methods []string //Methods
|
||||
AllowedHeaders []string //AllowHeaders
|
||||
ExposedHeaders []string //exposeHeader
|
||||
Credentials bool //cookie
|
||||
MaxAge int64 //缓存MaxAge
|
||||
PreflightContinue bool //在遇到Options的时候，继续，而不是返回请求
|
||||
OptionsSuccessStatus int //Options时，返回的状态码，默认204
|
||||
}
|
||||
|
||||
func configureOrigin(o *Options, c *gin.Context) map[string]string {
|
||||
reqOrigin := c.GetHeader("Origin")
|
||||
headers := make(map[string]string)
|
||||
if o.Origin == nil {
|
||||
headers["Access-Control-Allow-Origin"] = "*"
|
||||
} else {
|
||||
isAllowed := false
|
||||
for _, allowed := range *o.Origin {
|
||||
if allowed == reqOrigin {
|
||||
isAllowed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if isAllowed {
|
||||
headers["Access-Control-Allow-Origin"] = reqOrigin
|
||||
} else {
|
||||
headers["Access-Control-Allow-Origin"] = strconv.FormatBool(false)
|
||||
}
|
||||
headers["Vary"] = "Origin"
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
func configureMethods(o *Options) map[string]string {
|
||||
headers := make(map[string]string)
|
||||
if len(o.Methods) > 0 {
|
||||
headers["Access-Control-Allow-Methods"] = strings.ToUpper(strings.Join(o.Methods, ","))
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
func configureCredentials(o *Options) map[string]string {
|
||||
headers := make(map[string]string)
|
||||
if o.Credentials {
|
||||
headers["Access-Control-Allow-Credentials"] = strconv.FormatBool(true)
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
func configureAllowedHeaders(o *Options, c *gin.Context) map[string]string {
|
||||
headers := make(map[string]string)
|
||||
if len(o.AllowedHeaders) > 0 {
|
||||
headers["Access-Control-Allow-Headers"] = strings.Join(o.AllowedHeaders, ",")
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
func configureExposedHeaders(o *Options) map[string]string {
|
||||
var headers = make(map[string]string)
|
||||
if len(o.ExposedHeaders) > 0 {
|
||||
headers["Access-Control-Expose-Headers"] = strings.Join(o.ExposedHeaders, ",")
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
func configureMaxAge(o *Options) map[string]string {
|
||||
var headers = make(map[string]string)
|
||||
if o.MaxAge >= 0 {
|
||||
headers["Access-Control-Max-Age"] = strconv.FormatInt(o.MaxAge, 10)
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
func mergeMap(m ...map[string]string) map[string]string {
|
||||
r := make(map[string]string)
|
||||
for _, t := range m {
|
||||
for k, v := range t {
|
||||
r[k] = v
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func applyHeader(c *gin.Context, h map[string]string) {
|
||||
for k, v := range h {
|
||||
c.Header(k, v)
|
||||
}
|
||||
}
|
||||
|
||||
// Cors 跨域处理
|
||||
func Cors(o *Options) gin.HandlerFunc {
|
||||
if o.OptionsSuccessStatus == 0 {
|
||||
o.OptionsSuccessStatus = http.StatusNoContent
|
||||
}
|
||||
return func(c *gin.Context) {
|
||||
method := c.Request.Method
|
||||
//c.Header("Access-Control-Allow-Origin", "*")
|
||||
//c.Header("Access-Control-Allow-Headers", "Content-Type,AccessToken,X-CSRF-Token,Authorization,Token,Content-Length,Etag,Content-Range,Accept-Ranges,User-Agent,Range,Bucket,Content-Disposition,Signature,X-Forwarded-For,X-Real-Ip")
|
||||
if method == "OPTIONS" {
|
||||
h := mergeMap(
|
||||
configureOrigin(o, c),
|
||||
configureCredentials(o),
|
||||
configureMethods(o),
|
||||
configureAllowedHeaders(o, c),
|
||||
configureMaxAge(o),
|
||||
configureExposedHeaders(o),
|
||||
)
|
||||
applyHeader(c, h)
|
||||
if o.PreflightContinue {
|
||||
c.Next()
|
||||
} else {
|
||||
c.Header("Content-Length", "0")
|
||||
c.AbortWithStatus(o.OptionsSuccessStatus)
|
||||
}
|
||||
} else {
|
||||
h := mergeMap(
|
||||
configureOrigin(o, c),
|
||||
configureCredentials(o),
|
||||
configureExposedHeaders(o),
|
||||
)
|
||||
applyHeader(c, h)
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package ipblacklist
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/gin-gonic/gin"
|
||||
"net/http"
|
||||
"videoweb/app/service/configser"
|
||||
)
|
||||
|
||||
func IpBlackListMiddleware(ctx *gin.Context) {
|
||||
ip := ctx.ClientIP()
|
||||
if configser.CheckIpBlack(ip) {
|
||||
ctx.AbortWithStatusJSON(http.StatusUnauthorized, errors.New("login limit"))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package ipstat
|
||||
|
||||
import (
|
||||
"net"
|
||||
"time"
|
||||
"videoweb/pub/common/enum/redisconst"
|
||||
"videoweb/pub/common/ipaddr"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/utils"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/pub/redis"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// IpStatMiddleware 记录访客真实 IP，按天存入 Redis SET 实现去重计数。
|
||||
// key 格式：ipstat:{appId}:{yyyymmdd}，TTL 49 小时（保证昨日数据次日仍可读）。
|
||||
func IpStatMiddleware(ctx *gin.Context) {
|
||||
ctx.Next()
|
||||
|
||||
normalized := normalizeIP(ipaddr.ResolveClientIP(ctx))
|
||||
if normalized == "" {
|
||||
return
|
||||
}
|
||||
|
||||
date := time.Now().In(utils.LocBJ).Format("20060102")
|
||||
key := redisconst.IpStatKey(pubdef.AppId, date)
|
||||
|
||||
isNew, err := redisSAddNX(key, normalized)
|
||||
if err != nil {
|
||||
log.Warn("IpStatMiddleware SAdd failed", log.Any("key", key), log.E(err))
|
||||
return
|
||||
}
|
||||
if isNew {
|
||||
// 仅新增成员时刷新 TTL，避免每次请求都执行 EXPIRE
|
||||
if err := redis.Expire(key, redisconst.IpStatExpire); err != nil {
|
||||
log.Warn("IpStatMiddleware Expire failed", log.Any("key", key), log.E(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// redisSAddNX 添加成员并返回是否为新增（count > 0）
|
||||
func redisSAddNX(key string, member string) (bool, error) {
|
||||
err := redis.SAdd(key, member)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// normalizeIP 将原始 IP 标准化：
|
||||
// - 去掉 IPv4-mapped IPv6 前缀（::ffff:）
|
||||
// - 过滤私有/回环/链路本地地址
|
||||
// - 非法或私有 IP 返回空字符串
|
||||
func normalizeIP(raw string) string {
|
||||
ip := net.ParseIP(raw)
|
||||
if ip == nil {
|
||||
return ""
|
||||
}
|
||||
if v4 := ip.To4(); v4 != nil {
|
||||
ip = v4
|
||||
}
|
||||
if isPrivateIP(ip) {
|
||||
return ""
|
||||
}
|
||||
return ip.String()
|
||||
}
|
||||
|
||||
var privateRanges = []net.IPNet{
|
||||
parseCIDR("10.0.0.0/8"),
|
||||
parseCIDR("172.16.0.0/12"),
|
||||
parseCIDR("192.168.0.0/16"),
|
||||
parseCIDR("127.0.0.0/8"),
|
||||
parseCIDR("::1/128"),
|
||||
parseCIDR("fe80::/10"),
|
||||
parseCIDR("fc00::/7"),
|
||||
}
|
||||
|
||||
func parseCIDR(s string) net.IPNet {
|
||||
_, network, err := net.ParseCIDR(s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return *network
|
||||
}
|
||||
|
||||
func isPrivateIP(ip net.IP) bool {
|
||||
for _, r := range privateRanges {
|
||||
if r.Contains(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package monitor
|
||||
|
||||
import "time"
|
||||
|
||||
type APIStatis struct {
|
||||
//路径
|
||||
Path string `json:"path"`
|
||||
//消耗最大时间rawdata
|
||||
MaxTimeRawData string `json:"maxTimeRawData"`
|
||||
//调用次数
|
||||
CallNum int64 `json:"callNum"`
|
||||
//最小耗时
|
||||
MinTime int64 `json:"minTime"`
|
||||
//最大耗时
|
||||
MaxTime int64 `json:"maxTime"`
|
||||
//总耗时
|
||||
TotalTime int64 `json:"totalTime"`
|
||||
//每次调用时间
|
||||
TimePerCall int64 `json:"timePerCall"`
|
||||
//每秒调用次数
|
||||
CallPerSec int64 `json:"callPerSec"`
|
||||
//开始时间
|
||||
Start time.Time `json:"start"`
|
||||
//是否发送过报警
|
||||
Send int `json:"send"`
|
||||
}
|
||||
|
||||
// APIStatis 接口统计
|
||||
// 每天统计数
|
||||
type LogicStatis struct {
|
||||
//API总调用次数
|
||||
ApiCallNum int64
|
||||
//API调用总耗时
|
||||
ApiTotalTime int64
|
||||
//开始时间
|
||||
Start time.Time
|
||||
//接口访问次数
|
||||
ApisStatis map[string]*APIStatis
|
||||
}
|
||||
|
||||
type SPType string
|
||||
|
||||
const (
|
||||
SP_ApiCall SPType = "apicall"
|
||||
SP_GetStatis SPType = "getStatis"
|
||||
)
|
||||
|
||||
// StatisReq 统计请求数据格式
|
||||
type AppStatisReq struct {
|
||||
Time time.Time `json:"time"`
|
||||
//获取的api数量
|
||||
ApiNum int64 `json:"apiNum"`
|
||||
}
|
||||
|
||||
type StatisParam struct {
|
||||
//调用类型
|
||||
Type SPType
|
||||
Param1 string
|
||||
Param2 int64
|
||||
Param3 string
|
||||
}
|
||||
|
||||
var (
|
||||
hourStatises map[string]*LogicStatis
|
||||
muniteStatises map[string]*LogicStatis
|
||||
//api调用参数
|
||||
apiCallChan chan *StatisParam
|
||||
//退出
|
||||
)
|
||||
@@ -0,0 +1,262 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"sort"
|
||||
"time"
|
||||
"videoweb/pub/common/enum/consts"
|
||||
"videoweb/pub/common/enum/redisconst"
|
||||
"videoweb/pub/common/godef"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/srvdef"
|
||||
"videoweb/pub/common/tgbot"
|
||||
"videoweb/pub/redis"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// Check 运行监控检查
|
||||
func Check(ctx *gin.Context) {
|
||||
//查看是否有限制登陆
|
||||
ip := ctx.ClientIP()
|
||||
ifKey := redisconst.IpForbitKey(ip)
|
||||
retValue, _ := redis.Get(ifKey)
|
||||
if retValue != nil {
|
||||
//log.Info("Test Check", log.Any("ip", ip), log.Any("retValue", *retValue))
|
||||
ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
|
||||
"code": srvdef.ErrIpForbid,
|
||||
"msg": srvdef.ErrIpForbid.Msg(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Start timer
|
||||
start := time.Now()
|
||||
path := ctx.Request.URL.Path
|
||||
raw := ctx.Request.URL.RawQuery
|
||||
rawData, err := ctx.GetRawData()
|
||||
if err != nil {
|
||||
log.Info("Test get raw data error", log.E(err))
|
||||
}
|
||||
// 将取出来的执行条件放回ctx中
|
||||
ctx.Request.Body = ioutil.NopCloser(bytes.NewBuffer(rawData))
|
||||
//recover
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
buf := make([]byte, 2048)
|
||||
length := runtime.Stack(buf, false)
|
||||
info := string(buf[:length])
|
||||
log.Error("catch a unkown excetion", log.Any("excetion :", info))
|
||||
//发送tg消息
|
||||
msg := "monitor check crash,the stack:\n" + info
|
||||
tgbot.SendToProgram("API接口处理宕了", msg)
|
||||
}
|
||||
}()
|
||||
// Process request
|
||||
ctx.Next()
|
||||
}()
|
||||
// Stop timer
|
||||
stopTime := time.Now()
|
||||
latency := stopTime.Sub(start).Milliseconds()
|
||||
if raw != "" {
|
||||
path = path + "?" + raw
|
||||
}
|
||||
//添加记录
|
||||
//log.Info("Test", log.Any("rawData", rawData))
|
||||
rawString := string(rawData)
|
||||
addApiCall(path, int64(latency), rawString)
|
||||
//如果时间大于1秒发送tg消息
|
||||
if latency > 4000 {
|
||||
userId := ctx.GetInt64(consts.CtxUserID)
|
||||
msg := ""
|
||||
msg = msg + fmt.Sprintf("path:%s", path) + "\n"
|
||||
msg = msg + fmt.Sprintf("userId:%d", userId) + "\n"
|
||||
msg = msg + fmt.Sprintf("userIp:%s", ip) + "\n"
|
||||
msg = msg + fmt.Sprintf("rawData:%s", string(rawData)) + "\n"
|
||||
msg = msg + fmt.Sprintf("time:%d", latency) + "\n"
|
||||
tgbot.SendToProgram("API接口调用超时", msg)
|
||||
}
|
||||
//log.Info("monitor:Check", log.Any("latency", latency), log.Any("path", path))
|
||||
}
|
||||
|
||||
func init() {
|
||||
hourStatises = make(map[string]*LogicStatis)
|
||||
muniteStatises = make(map[string]*LogicStatis)
|
||||
apiCallChan = make(chan *StatisParam, 1024)
|
||||
godef.Go(loop)
|
||||
log.Info("(monitor:init) init success.")
|
||||
}
|
||||
|
||||
func loop() {
|
||||
log.Info("(monitor:loop) start...")
|
||||
defer log.Info("monitor:loop) end")
|
||||
var param *StatisParam
|
||||
for {
|
||||
select {
|
||||
case param = <-apiCallChan:
|
||||
onStatis(param)
|
||||
default:
|
||||
time.Sleep(time.Millisecond * 10)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func onStatis(param *StatisParam) {
|
||||
if param == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
hourStr := now.Format("2006-01-02 15")
|
||||
hourStatis, hourOk := hourStatises[hourStr]
|
||||
if !hourOk {
|
||||
hourStatis = &LogicStatis{
|
||||
ApisStatis: make(map[string]*APIStatis),
|
||||
}
|
||||
hourStatises[hourStr] = hourStatis
|
||||
}
|
||||
muniteStr := now.Format("2006-01-02 15:04")
|
||||
minuteStatis, minuteOk := muniteStatises[muniteStr]
|
||||
if !minuteOk {
|
||||
minuteStatis = &LogicStatis{
|
||||
ApisStatis: make(map[string]*APIStatis),
|
||||
}
|
||||
muniteStatises[muniteStr] = minuteStatis
|
||||
}
|
||||
onStatisApiCall(hourStr, param, hourStatis, 10000000)
|
||||
onStatisApiCall(muniteStr, param, minuteStatis, 20000)
|
||||
//如果新的一小时,发送上一小时数据
|
||||
if !hourOk {
|
||||
//清空分钟数据
|
||||
muniteStatises = make(map[string]*LogicStatis)
|
||||
lastHour := now.Add(-time.Hour).Format("2006-01-02 15")
|
||||
hourStatis, hourOk = hourStatises[lastHour]
|
||||
if hourOk {
|
||||
msg := fmt.Sprintf("统计时间:%s\n", lastHour)
|
||||
msg += formatStatis(hourStatis)
|
||||
tgbot.SendToProgram("API接口调用统计", msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func formatStatis(hourStatis *LogicStatis) string {
|
||||
apiStatises := make([]*APIStatis, 0)
|
||||
for _, static := range hourStatis.ApisStatis {
|
||||
apiStatises = append(apiStatises, static)
|
||||
}
|
||||
sort.Slice(apiStatises, func(i, j int) bool {
|
||||
return apiStatises[i].CallNum > apiStatises[j].CallNum
|
||||
})
|
||||
//发送数据
|
||||
msg := ""
|
||||
maxlen := len(apiStatises)
|
||||
if maxlen > 10 {
|
||||
maxlen = 10
|
||||
}
|
||||
for i := 0; i < maxlen; i++ {
|
||||
apiStatis := apiStatises[i]
|
||||
msg += formatApiStatis(apiStatis)
|
||||
}
|
||||
timePerCall := int64(0)
|
||||
callPerSec := int64(0)
|
||||
if hourStatis.ApiCallNum > 0 {
|
||||
timePerCall = hourStatis.ApiTotalTime / hourStatis.ApiCallNum
|
||||
diffSecdonds := int64(time.Now().Sub(hourStatis.Start).Seconds())
|
||||
if diffSecdonds > 0 {
|
||||
callPerSec = hourStatis.ApiCallNum / diffSecdonds
|
||||
}
|
||||
}
|
||||
msg += fmt.Sprintf("ApiCallNum:%d \n", hourStatis.ApiCallNum)
|
||||
msg += fmt.Sprintf("ApiTotalTime:%d \n", hourStatis.ApiTotalTime)
|
||||
msg += fmt.Sprintf("ApiTimePerCall:%d \n", timePerCall)
|
||||
msg += fmt.Sprintf("ApiCallPerSec:%d \n", callPerSec)
|
||||
return msg
|
||||
}
|
||||
|
||||
func formatApiStatis(apiStatis *APIStatis) string {
|
||||
msg := ""
|
||||
if apiStatis.CallNum > 0 {
|
||||
apiStatis.TimePerCall = apiStatis.TotalTime / apiStatis.CallNum
|
||||
diffSeconds := int64(time.Now().Sub(apiStatis.Start).Seconds())
|
||||
if diffSeconds > 0 {
|
||||
apiStatis.CallPerSec = apiStatis.CallNum / diffSeconds
|
||||
}
|
||||
}
|
||||
|
||||
statisStr := fmt.Sprintf(`path %s:
|
||||
MaxRawData:%s
|
||||
CallNum:%d;MinTime:%d;MaxTime:%d;TotalTime:%d;TimePerCall:%d;CallPerSec:%d`,
|
||||
apiStatis.Path,
|
||||
apiStatis.MaxTimeRawData,
|
||||
apiStatis.CallNum,
|
||||
apiStatis.MinTime,
|
||||
apiStatis.MaxTime,
|
||||
apiStatis.TotalTime,
|
||||
apiStatis.TimePerCall,
|
||||
apiStatis.CallPerSec)
|
||||
msg += statisStr
|
||||
msg += "\n---------------------------------------------------\n"
|
||||
return msg
|
||||
}
|
||||
|
||||
// api统计
|
||||
func onStatisApiCall(dateStr string, param *StatisParam,
|
||||
logicStatis *LogicStatis, maxCallNum int64) {
|
||||
logicStatis.ApiCallNum++
|
||||
logicStatis.ApiTotalTime += param.Param2
|
||||
apiStatis, ok := logicStatis.ApisStatis[param.Param1]
|
||||
if !ok {
|
||||
apiStatis = &APIStatis{
|
||||
Path: param.Param1,
|
||||
MaxTimeRawData: param.Param3,
|
||||
CallNum: 0,
|
||||
MinTime: math.MaxInt64,
|
||||
MaxTime: 0,
|
||||
TotalTime: 0,
|
||||
Start: time.Now(),
|
||||
Send: 0,
|
||||
}
|
||||
logicStatis.ApisStatis[param.Param1] = apiStatis
|
||||
}
|
||||
apiStatis.CallNum++
|
||||
apiStatis.TotalTime += param.Param2
|
||||
if param.Param2 > apiStatis.MaxTime {
|
||||
apiStatis.MaxTime = param.Param2
|
||||
apiStatis.MaxTimeRawData = param.Param3
|
||||
}
|
||||
if param.Param2 < apiStatis.MinTime {
|
||||
apiStatis.MinTime = param.Param2
|
||||
}
|
||||
if apiStatis.CallNum > maxCallNum && apiStatis.Send < 4 {
|
||||
apiStatis.Send++
|
||||
msg := fmt.Sprintf("接口%s每分钟调用超过最大次数%d\n", apiStatis.Path, maxCallNum)
|
||||
msg += formatApiStatis(apiStatis)
|
||||
tgbot.SendToProgram("API调用警告", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func addApiCall(path string, latency int64, rawData string) {
|
||||
param := &StatisParam{
|
||||
Type: SP_ApiCall,
|
||||
Param1: path,
|
||||
Param2: latency,
|
||||
Param3: rawData,
|
||||
}
|
||||
pushParam(param)
|
||||
}
|
||||
|
||||
func pushParam(param *StatisParam) {
|
||||
//no block
|
||||
select {
|
||||
case apiCallChan <- param:
|
||||
default:
|
||||
info := "(monitor:pushParam) failed,chan is full."
|
||||
log.Info(info)
|
||||
tgbot.SendToProgram("monitor pushParam failed", info)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package reptilerecord
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"strings"
|
||||
"videoweb/pub/common/log"
|
||||
)
|
||||
|
||||
const ctxKeyIsBot = "isBot"
|
||||
|
||||
var (
|
||||
botPatterns = map[string]string{
|
||||
"Googlebot": "Googlebot",
|
||||
"Bingbot": "Bingbot",
|
||||
"Slurp": "Slurp",
|
||||
"DuckDuckBot": "DuckDuckBot",
|
||||
"Baiduspider": "Baiduspider",
|
||||
"YandexBot": "YandexBot",
|
||||
"Sogou": "Sogou",
|
||||
"Exabot": "Exabot",
|
||||
"facebot": "facebot",
|
||||
"facebookexternalhit": "facebookexternalhit",
|
||||
}
|
||||
)
|
||||
|
||||
func ReptileRecord(c *gin.Context) {
|
||||
ua := c.Request.UserAgent()
|
||||
botName := identifyBot(ua)
|
||||
if botName != "" {
|
||||
c.Set(ctxKeyIsBot, true)
|
||||
log.Info("爬虫访问:", log.Any("botName", botName), log.Any("ua", ua), log.Any("path", c.Request.URL.Path), log.Any("ip", c.ClientIP()))
|
||||
}
|
||||
}
|
||||
|
||||
// IsBot 返回当前请求是否来自已知搜索引擎爬虫
|
||||
func IsBot(c *gin.Context) bool {
|
||||
v, exists := c.Get(ctxKeyIsBot)
|
||||
return exists && v.(bool)
|
||||
}
|
||||
|
||||
func identifyBot(userAgent string) string {
|
||||
for name, pattern := range botPatterns {
|
||||
if strings.Contains(userAgent, pattern) {
|
||||
return name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package templatecache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/patrickmn/go-cache"
|
||||
)
|
||||
|
||||
const (
|
||||
maxCacheItems = 2000
|
||||
maxCacheBytes = 64 << 20
|
||||
)
|
||||
|
||||
var (
|
||||
Cache *cache.Cache
|
||||
cacheBytes int64
|
||||
)
|
||||
|
||||
// 章节详情、资讯详情等 ID 空间过大，禁止进进程缓存。
|
||||
var highCardinalityContains = []string{
|
||||
"/chapter/info",
|
||||
"/consultinginfo/",
|
||||
}
|
||||
|
||||
func init() {
|
||||
Cache = cache.New(5*time.Minute, 5*time.Minute)
|
||||
Cache.OnEvicted(func(_ string, v interface{}) {
|
||||
if b, ok := v.([]byte); ok {
|
||||
atomic.AddInt64(&cacheBytes, -int64(len(b)))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func shouldBypassCache(path string) bool {
|
||||
if path == "" {
|
||||
return false
|
||||
}
|
||||
if i := strings.IndexByte(path, '?'); i >= 0 {
|
||||
path = path[:i]
|
||||
}
|
||||
for _, part := range highCardinalityContains {
|
||||
if strings.Contains(path, part) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func cacheBudgetAllows(n int) bool {
|
||||
if n <= 0 {
|
||||
return false
|
||||
}
|
||||
if Cache.ItemCount() >= maxCacheItems {
|
||||
return false
|
||||
}
|
||||
return atomic.LoadInt64(&cacheBytes)+int64(n) <= maxCacheBytes
|
||||
}
|
||||
|
||||
type templateResponseWriter struct {
|
||||
gin.ResponseWriter
|
||||
body *bytes.Buffer
|
||||
}
|
||||
|
||||
func (w *templateResponseWriter) Write(b []byte) (int, error) {
|
||||
w.body.Write(b)
|
||||
return w.ResponseWriter.Write(b)
|
||||
}
|
||||
|
||||
func WithCache(duration time.Duration) gin.HandlerFunc {
|
||||
return func(ctx *gin.Context) {
|
||||
if ctx.Request.Method != "GET" {
|
||||
ctx.Next()
|
||||
return
|
||||
}
|
||||
if shouldBypassCache(ctx.Request.URL.Path) {
|
||||
ctx.Next()
|
||||
return
|
||||
}
|
||||
|
||||
cacheKey := ctx.Request.URL.String()
|
||||
if cacheData, exists := Cache.Get(cacheKey); exists {
|
||||
if item, ok := cacheData.([]byte); ok && len(item) > 0 {
|
||||
ctx.Data(http.StatusOK, "application/json; charset=utf-8", item)
|
||||
ctx.Abort()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
writer := &templateResponseWriter{
|
||||
body: bytes.NewBuffer(nil),
|
||||
ResponseWriter: ctx.Writer,
|
||||
}
|
||||
ctx.Writer = writer
|
||||
ctx.Next()
|
||||
|
||||
if ctx.Writer.Status() != 200 {
|
||||
return
|
||||
}
|
||||
raw := writer.body.Bytes()
|
||||
if !cacheBudgetAllows(len(raw)) {
|
||||
return
|
||||
}
|
||||
copied := append([]byte(nil), raw...)
|
||||
atomic.AddInt64(&cacheBytes, int64(len(copied)))
|
||||
cacheDuration := duration
|
||||
if cacheDuration == 0 {
|
||||
cacheDuration = time.Minute
|
||||
}
|
||||
Cache.Set(cacheKey, copied, cacheDuration)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package validator
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gin-gonic/gin/binding"
|
||||
"github.com/go-playground/locales/en"
|
||||
ut "github.com/go-playground/universal-translator"
|
||||
"gopkg.in/go-playground/validator.v9"
|
||||
en_translations "gopkg.in/go-playground/validator.v9/translations/en"
|
||||
)
|
||||
|
||||
var (
|
||||
ValidatorUni *ut.UniversalTranslator
|
||||
)
|
||||
|
||||
type defaultValidator struct {
|
||||
once sync.Once
|
||||
validate *validator.Validate
|
||||
}
|
||||
|
||||
var v binding.StructValidator = &defaultValidator{}
|
||||
|
||||
func init() {
|
||||
v.Engine()
|
||||
}
|
||||
|
||||
func (v *defaultValidator) ValidateStruct(obj interface{}) error {
|
||||
|
||||
if kindOfData(obj) == reflect.Struct {
|
||||
|
||||
v.lazyinit()
|
||||
|
||||
if err := v.validate.Struct(obj); err != nil {
|
||||
return error(err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *defaultValidator) Engine() interface{} {
|
||||
v.lazyinit()
|
||||
return v.validate
|
||||
}
|
||||
|
||||
func (v *defaultValidator) lazyinit() {
|
||||
v.once.Do(func() {
|
||||
v.validate = validator.New()
|
||||
|
||||
// 依旧使用bingding tag兼容v8
|
||||
v.validate.SetTagName("binding")
|
||||
|
||||
// 错误信息转换
|
||||
en := en.New()
|
||||
ValidatorUni = ut.New(en, en)
|
||||
trans, _ := ValidatorUni.GetTranslator("en")
|
||||
en_translations.RegisterDefaultTranslations(v.validate, trans)
|
||||
|
||||
v.validate.RegisterTagNameFunc(func(fld reflect.StructField) string {
|
||||
name := strings.SplitN(fld.Tag.Get("comment"), ",", 2)[0]
|
||||
if name == "-" {
|
||||
return ""
|
||||
}
|
||||
return name
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func kindOfData(data interface{}) reflect.Kind {
|
||||
|
||||
value := reflect.ValueOf(data)
|
||||
valueType := value.Kind()
|
||||
|
||||
if valueType == reflect.Ptr {
|
||||
valueType = value.Elem().Kind()
|
||||
}
|
||||
return valueType
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package validator
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin/binding"
|
||||
)
|
||||
|
||||
// 替换gin 默认v8验证器
|
||||
// 改为v9
|
||||
func init() {
|
||||
binding.Validator = new(defaultValidator)
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package verfyparam
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
var (
|
||||
// 白名单，不需要验证token的api
|
||||
whitelist = map[string]bool{}
|
||||
)
|
||||
|
||||
func VerifyReplayAttackRequest() gin.HandlerFunc {
|
||||
return func(ctx *gin.Context) {
|
||||
for url, ok := range whitelist {
|
||||
if ok && strings.HasPrefix(ctx.Request.URL.Path, url) {
|
||||
return
|
||||
}
|
||||
}
|
||||
ctx.Next()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package visit
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// 用户访问中间件
|
||||
// 记录用户在线量
|
||||
// 按5分钟纬度记录到redis中
|
||||
func Visit(ctx *gin.Context) {
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user