360 lines
9.0 KiB
Go
360 lines
9.0 KiB
Go
package truthutil
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"regexp"
|
||
"strings"
|
||
"time"
|
||
"unicode"
|
||
|
||
"github.com/mozillazg/go-pinyin"
|
||
|
||
"91porn-server/app/appg"
|
||
"91porn-server/common/constant/redisconst"
|
||
"91porn-server/common/localcache"
|
||
"91porn-server/common/log"
|
||
"91porn-server/models/v/filtermod"
|
||
)
|
||
|
||
// CheckIsValid 判断评论、私信等内容是否合法
|
||
func CheckIsValid(a string, checkType int) bool {
|
||
// 忽略emo表情 参考：https://apps.timwhitlock.info/emoji/tables/unicode#block-6c-other-additional-symbols
|
||
reg := regexp.MustCompile(`[\x{1F600}-\x{1F64F}\x{1F444}-\x{1F445}\x{1F488}-\x{1F48B}]+`)
|
||
a = reg.ReplaceAllString(a, "")
|
||
|
||
// 检查白名单
|
||
if valid := checkWhiteListChars(a); !valid {
|
||
return false
|
||
}
|
||
// 检查后台敏感词
|
||
if valid := checkSensitiveWords(a, checkType); !valid {
|
||
return false
|
||
}
|
||
// 检查url和数字
|
||
if valid := checkURIAndNumber(a); !valid {
|
||
return false
|
||
}
|
||
// 检查评论频率
|
||
if valid := checkRepeat(a); !valid {
|
||
return false
|
||
}
|
||
|
||
return true
|
||
}
|
||
|
||
// 仅允许常规字符
|
||
func checkWhiteListChars(s string) bool {
|
||
// 全符号文本
|
||
cv, err := NewCommentValidator(`^[\d\p{P}\p{S}\s]*$`)
|
||
if err != nil {
|
||
return false
|
||
}
|
||
valid := cv.Validate(s)
|
||
if valid {
|
||
return false
|
||
}
|
||
|
||
cv, err = NewCommentValidator("^[A-Za-z0-9*!,.?，。？！、 \u4e00-\u9fa5]+$")
|
||
if err != nil {
|
||
log.Error(fmt.Sprintf("checkWhiteListChars Invalid pattern err:%v", err))
|
||
return false
|
||
}
|
||
valid = cv.Validate(s)
|
||
if !valid {
|
||
return false
|
||
}
|
||
|
||
return true
|
||
}
|
||
|
||
func checkSensitiveWords(a string, checkType int) bool {
|
||
// 获取缓存数据
|
||
str, err := appg.Redis.Get(redisconst.SensitiveWordsCache)
|
||
if err != nil {
|
||
log.Warn(fmt.Sprintf("redis get SensitiveWords:%v", err))
|
||
}
|
||
|
||
var sensitiveWords []string
|
||
|
||
if str != nil {
|
||
if err = json.Unmarshal([]byte(*str), &sensitiveWords); err != nil {
|
||
log.Warn(fmt.Sprintf("redis get SensitiveWords parse err:%v", err))
|
||
}
|
||
} else {
|
||
words, err := filtermod.GetAllFilterWords()
|
||
if err != nil || len(words) <= 0 {
|
||
return true
|
||
}
|
||
|
||
for _, w := range words {
|
||
sensitiveWords = append(sensitiveWords, w.Word)
|
||
}
|
||
|
||
if len(sensitiveWords) > 0 {
|
||
// 放入缓存
|
||
bytes, _ := json.Marshal(sensitiveWords)
|
||
err := appg.Redis.Set(redisconst.SensitiveWordsCache, bytes, 10*time.Minute)
|
||
if err != nil {
|
||
log.Warn(fmt.Sprintf("redis set SensitiveWords err:%v", err))
|
||
}
|
||
}
|
||
}
|
||
|
||
a = strings.ReplaceAll(a, " ", "")
|
||
a = strings.ReplaceAll(a, "\n", "")
|
||
a = strings.ToUpper(a)
|
||
for _, s := range sensitiveWords {
|
||
filterStr := strings.ReplaceAll(s, " ", "")
|
||
filterStr = strings.ReplaceAll(filterStr, "\n", "")
|
||
filterStr = strings.ToUpper(filterStr)
|
||
if checkType == 0 {
|
||
// 个人用户名,简介等
|
||
if strings.Contains(a, filterStr) {
|
||
return false
|
||
}
|
||
} else {
|
||
// 用于评论
|
||
// 2026-3 改为精确匹配
|
||
if a == filterStr {
|
||
return false
|
||
}
|
||
}
|
||
}
|
||
|
||
return true
|
||
}
|
||
|
||
// checkURIAndNumber 检查qq号或网址
|
||
func checkURIAndNumber(a string) bool {
|
||
a = strings.ReplaceAll(a, "匚", "c")
|
||
a = strings.ReplaceAll(a, "匸", "c")
|
||
a = strings.ReplaceAll(a, "丅", "t")
|
||
a = strings.ReplaceAll(a, "卩", "p")
|
||
a = strings.ReplaceAll(a, "卪", "p")
|
||
a = strings.ReplaceAll(a, "讠", "i")
|
||
a = strings.ReplaceAll(a, "丶", ".")
|
||
a = strings.ReplaceAll(a, "、", ".")
|
||
a = strings.ReplaceAll(a, ",", ".")
|
||
a = strings.ReplaceAll(a, "，", ".")
|
||
a = strings.ReplaceAll(a, "。", ".")
|
||
a = strings.ReplaceAll(a, "俛", "免")
|
||
eRegexp := regexp.MustCompile("[^\u4e00-\u9fa50-9a-zA-Z.。+]+")
|
||
effective := eRegexp.ReplaceAllString(a, "")
|
||
|
||
ra := []rune(effective)
|
||
final := ""
|
||
|
||
args := pinyin.NewArgs()
|
||
args.Fallback = func(r rune, a pinyin.Args) []string {
|
||
return []string{string(r)}
|
||
}
|
||
|
||
pys := pinyin.LazyConvert(effective, &args)
|
||
numMap := make(map[int]bool)
|
||
var diffNumCount int
|
||
var haveJia bool
|
||
var haveWei bool
|
||
var yue bool
|
||
var pao bool
|
||
var wang bool
|
||
var mian bool
|
||
var fei bool
|
||
for i, py := range pys {
|
||
c := string(ra[i])
|
||
n := -1
|
||
switch py {
|
||
case "ling", "ning", "lin", "nin", "O", "0":
|
||
n = 0
|
||
if py == "0" || py == "o" || py == "O" {
|
||
c = "o"
|
||
} else {
|
||
c = "0"
|
||
}
|
||
case "yi", "1":
|
||
n = 1
|
||
if py == "yi" {
|
||
c = "e"
|
||
}
|
||
case "er", "2":
|
||
n = 2
|
||
c = "2"
|
||
case "san", "sa", "shan", "3":
|
||
n = 3
|
||
c = "3"
|
||
case "si", "shi", "4":
|
||
n = 4
|
||
c = "4"
|
||
case "wu", "5":
|
||
n = 5
|
||
c = "5"
|
||
case "liu", "lu", "niu", "nu", "6":
|
||
n = 6
|
||
c = "6"
|
||
case "qi", "7":
|
||
n = 7
|
||
c = "7"
|
||
case "ba", "pa", "8":
|
||
n = 8
|
||
c = "8"
|
||
case "jiu", "9":
|
||
n = 9
|
||
c = "9"
|
||
case "dian":
|
||
c = "."
|
||
// "xi" 是常见汉字读音，不转成 "c"，避免“嘻嘻”等正常内容被误判为 .cc 域名。
|
||
case "se":
|
||
c = "c"
|
||
case "ti":
|
||
c = "t"
|
||
case "en", "eng":
|
||
c = "n"
|
||
case "ou":
|
||
c = "o"
|
||
case "pi":
|
||
c = "p"
|
||
case "jia", "+":
|
||
haveJia = true
|
||
case "wei", "v", "V", "w", "W", "qun", "qiong", "kou", "qiu", "q", "Q":
|
||
haveWei = true
|
||
case "wang":
|
||
wang = true
|
||
case "yue", "yo":
|
||
yue = true
|
||
case "pao", "bao":
|
||
pao = true
|
||
case "mian":
|
||
mian = true
|
||
case "fei":
|
||
fei = true
|
||
}
|
||
|
||
if n >= 0 {
|
||
if _, ok := numMap[n]; !ok {
|
||
numMap[n] = true
|
||
diffNumCount++
|
||
}
|
||
}
|
||
|
||
final += c
|
||
}
|
||
|
||
lRegexp := regexp.MustCompile(`[^a-zA-Z0-57-9]+`)
|
||
letter := lRegexp.ReplaceAllString(final, "")
|
||
//fmt.Println(effective)
|
||
//fmt.Println(final)
|
||
//fmt.Println(alpha)
|
||
// 数字检查
|
||
if diffNumCount >= 4 || len(letter) >= 5 {
|
||
return false
|
||
}
|
||
if haveJia && haveWei {
|
||
return false
|
||
}
|
||
|
||
alphaRegexp := regexp.MustCompile(`[^a-zA-Z0-9.]+`)
|
||
alpha := alphaRegexp.ReplaceAllString(final, "")
|
||
// 网址检查
|
||
if r := regexp.MustCompile(`(?i)\w*(aaa|abb|abc|ac|aco|ad|ads|ae|aeg|af|afl|ag|ai|aig|al|am|anz|ao|aol|app|aq|ar|art|as|at|au|aw|aws|ax|axa|az|ba|bar|bb|bbc|bbt|bcg|bcn|bd|be|bet|bf|bg|bh|bi|bid|bio|biz|bj|bm|bms|bmw|bn|bo|bom|boo|bot|box|br|bs|bt|buy|bv|bw|by|bz|bzh|ca|cab|cal|cam|car|cat|cba|cbn|cc|cd|ceo|cf|cfa|cfd|cg|ch|ci|ck|cl|cm|cn|co|com|cpa|cr|crs|cu|cv|cw|cx|cy|cz|dad|day|dds|de|dev|dhl|diy|dj|dk|dm|dnp|do|dog|dot|dtv|dvr|dz|eat|ec|eco|edu|ee|eg|er|es|esq|et|eu|eus|fan|fi|fit|fj|fk|fly|fm|fo|foo|fox|fr|frl|ftr|fun|fyi|hbo|hiv|hk|hkt|hm|hn|hot|how|hr|ht|hu|ibm|ice|icu|id|ie|ifm|il|im|in|inc|ing|ink|int|io|iq|ir|is|ist|it|itv|jcb|je|jio|jll|jm|jmp|jnj|jo|jot|joy|jp|ke|kfh|kg|kh|ki|kia|kim|km|kn|kp|kpn|kr|krd|kw|ky|kz|la|lat|law|lb|lc|lds|li|lk|llc|llp|lol|lpl|lr|ls|lt|ltd|lu|lv|ly|ma|man|map|mba|mc|md|me|med|men|mg|mh|mil|mit|mk|ml|mlb|mls|mm|mma|mn|mo|moe|moi|mom|mov|mp|mq|mr|ms|msd|mt|mtn|mtr|mu|mv|mw|mx|my|mz|na|nab|nba|nc|ne|nec|net|new|nf|nfl|ng|ngo|nhk|ni|nl|no|now|np|nr|nra|nrw|ntt|nu|nyc|nz|obi|om|one|ong|onl|ooo|org|ott|ovh|pa|pay|pe|pet|pf|pg|ph|phd|pid|pin|pk|pl|pm|pn|pnc|pr|pro|pru|ps|pt|pub|pw|pwc|py|qa|re|red|ren|ril|rio|rip|ro|rs|ru|run|rw|rwe|sa|sap|sas|sb|sbi|sbs|sc|scb|sd|se|sew|sex|sfr|sg|sh|si|sj|sk|ski|sky|sl|sm|sn|so|soy|spa|sr|srl|ss|st|stc|su|sv|sx|sy|sz|tab|tax|tc|tci|td|tdk|tel|tf|tg|th|thd|tj|tjx|tk|tl|tm|tn|to|top|tr|trv|tt|tui|tv|tvs|tw|tz|ua|ubs|ug|uk|uno|uol|ups|us|uy|uz|va|vc|ve|vet|vg|vi|vig|vin|vip|vn|vu|wed|wf|win|wme|wow|ws|wtc|wtf|xin|xxx|xyz|ye|you|yt|yun|za|zip|zm|zw)`); r != nil {
|
||
// 匹配成功
|
||
if r.MatchString(alpha) {
|
||
return false
|
||
}
|
||
}
|
||
if r := regexp.MustCompile(`(?i)\w+\.[0-1a-z]{2,}`); r != nil {
|
||
// 匹配成功
|
||
if r.MatchString(alpha) {
|
||
return false
|
||
}
|
||
}
|
||
|
||
// 综合检查
|
||
if (wang || yue || pao || (mian && fei)) && len(letter) >= 3 {
|
||
return false
|
||
}
|
||
|
||
return true
|
||
}
|
||
|
||
// checkRepeat 30分钟内超过5条一样的 拉入12小时黑名单
|
||
func checkRepeat(a string) bool {
|
||
numRegexp := regexp.MustCompile("[^\u4e00-\u9fa5a-zA-Z]+")
|
||
a = numRegexp.ReplaceAllString(a, "")
|
||
|
||
var mLimit, cLimit = 30, 5
|
||
var black = 12
|
||
var hasHan bool
|
||
//var hasAlpha bool
|
||
rmap := make(map[int32]bool)
|
||
var rc int
|
||
for _, r := range a {
|
||
if unicode.Is(unicode.Han, r) {
|
||
hasHan = true
|
||
}
|
||
//if unicode.IsDigit(r) || isEnLetter(r) {
|
||
// hasAlpha = true
|
||
//}
|
||
if _, ok := rmap[r]; !ok {
|
||
rmap[r] = true
|
||
rc++
|
||
}
|
||
}
|
||
|
||
// 必须包含汉字并且字母或数字 并且5个以上不重复字符
|
||
if !hasHan || rc < 5 {
|
||
return true
|
||
}
|
||
|
||
// 记录评论map
|
||
r := []rune(a)
|
||
if len(r) > 30 {
|
||
s1 := r[:30]
|
||
a = string(s1)
|
||
}
|
||
|
||
bKey := "cmt-b-" + a
|
||
key := "cmt-" + a
|
||
// 查看黑名单
|
||
_, ok := localcache.C.Get(bKey)
|
||
if ok {
|
||
return false
|
||
}
|
||
// 记录次数
|
||
c, ok2 := localcache.C.Get(key)
|
||
if !ok2 {
|
||
localcache.C.Set(key, 1, time.Duration(mLimit)*time.Minute)
|
||
} else {
|
||
// 超过次数加入黑名单
|
||
ci := c.(int)
|
||
if ci+1 >= cLimit {
|
||
localcache.C.Set(bKey, true, time.Duration(black)*time.Hour)
|
||
localcache.C.Delete(key)
|
||
return false
|
||
}
|
||
_ = localcache.C.Increment(key, 1)
|
||
}
|
||
|
||
return true
|
||
}
|
||
|
||
// 判断英文字母
|
||
func isEnLetter(s rune) bool {
|
||
return (s >= 97 && s <= 122) || (s >= 65 && s <= 90)
|
||
}
|
||
|
||
type CommentValidator struct {
|
||
pattern *regexp.Regexp
|
||
}
|
||
|
||
func NewCommentValidator(pattern string) (*CommentValidator, error) {
|
||
re, err := regexp.Compile(pattern)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return &CommentValidator{pattern: re}, nil
|
||
}
|
||
|
||
func (cv *CommentValidator) Validate(comment string) bool {
|
||
return cv.pattern.MatchString(comment)
|
||
}
|