Initial commit

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

common/truthutil/truthutil.go

@@ -0,0 +1,359 @@
+package truthutil
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/mozillazg/go-pinyin"
+
+	"91porn-server/app/appg"
+	"91porn-server/common/constant/redisconst"
+	"91porn-server/common/localcache"
+	"91porn-server/common/log"
+	"91porn-server/models/v/filtermod"
+)
+
+// CheckIsValid 判断评论、私信等内容是否合法
+func CheckIsValid(a string, checkType int) bool {
+	// 忽略emo表情 参考：https://apps.timwhitlock.info/emoji/tables/unicode#block-6c-other-additional-symbols
+	reg := regexp.MustCompile(`[\x{1F600}-\x{1F64F}\x{1F444}-\x{1F445}\x{1F488}-\x{1F48B}]+`)
+	a = reg.ReplaceAllString(a, "")
+
+	// 检查白名单
+	if valid := checkWhiteListChars(a); !valid {
+		return false
+	}
+	// 检查后台敏感词
+	if valid := checkSensitiveWords(a, checkType); !valid {
+		return false
+	}
+	// 检查url和数字
+	if valid := checkURIAndNumber(a); !valid {
+		return false
+	}
+	// 检查评论频率
+	if valid := checkRepeat(a); !valid {
+		return false
+	}
+
+	return true
+}
+
+// 仅允许常规字符
+func checkWhiteListChars(s string) bool {
+	// 全符号文本
+	cv, err := NewCommentValidator(`^[\d\p{P}\p{S}\s]*$`)
+	if err != nil {
+		return false
+	}
+	valid := cv.Validate(s)
+	if valid {
+		return false
+	}
+
+	cv, err = NewCommentValidator("^[A-Za-z0-9*!,.?，。？！、 \u4e00-\u9fa5]+$")
+	if err != nil {
+		log.Error(fmt.Sprintf("checkWhiteListChars Invalid pattern err:%v", err))
+		return false
+	}
+	valid = cv.Validate(s)
+	if !valid {
+		return false
+	}
+
+	return true
+}
+
+func checkSensitiveWords(a string, checkType int) bool {
+	// 获取缓存数据
+	str, err := appg.Redis.Get(redisconst.SensitiveWordsCache)
+	if err != nil {
+		log.Warn(fmt.Sprintf("redis get SensitiveWords:%v", err))
+	}
+
+	var sensitiveWords []string
+
+	if str != nil {
+		if err = json.Unmarshal([]byte(*str), &sensitiveWords); err != nil {
+			log.Warn(fmt.Sprintf("redis get SensitiveWords parse err:%v", err))
+		}
+	} else {
+		words, err := filtermod.GetAllFilterWords()
+		if err != nil || len(words) <= 0 {
+			return true
+		}
+
+		for _, w := range words {
+			sensitiveWords = append(sensitiveWords, w.Word)
+		}
+
+		if len(sensitiveWords) > 0 {
+			// 放入缓存
+			bytes, _ := json.Marshal(sensitiveWords)
+			err := appg.Redis.Set(redisconst.SensitiveWordsCache, bytes, 10*time.Minute)
+			if err != nil {
+				log.Warn(fmt.Sprintf("redis set SensitiveWords err:%v", err))
+			}
+		}
+	}
+
+	a = strings.ReplaceAll(a, " ", "")
+	a = strings.ReplaceAll(a, "\n", "")
+	a = strings.ToUpper(a)
+	for _, s := range sensitiveWords {
+		filterStr := strings.ReplaceAll(s, " ", "")
+		filterStr = strings.ReplaceAll(filterStr, "\n", "")
+		filterStr = strings.ToUpper(filterStr)
+		if checkType == 0 {
+			// 个人用户名,简介等
+			if strings.Contains(a, filterStr) {
+				return false
+			}
+		} else {
+			// 用于评论
+			// 2026-3 改为精确匹配
+			if a == filterStr {
+				return false
+			}
+		}
+	}
+
+	return true
+}
+
+// checkURIAndNumber 检查qq号或网址
+func checkURIAndNumber(a string) bool {
+	a = strings.ReplaceAll(a, "匚", "c")
+	a = strings.ReplaceAll(a, "匸", "c")
+	a = strings.ReplaceAll(a, "丅", "t")
+	a = strings.ReplaceAll(a, "卩", "p")
+	a = strings.ReplaceAll(a, "卪", "p")
+	a = strings.ReplaceAll(a, "讠", "i")
+	a = strings.ReplaceAll(a, "丶", ".")
+	a = strings.ReplaceAll(a, "、", ".")
+	a = strings.ReplaceAll(a, ",", ".")
+	a = strings.ReplaceAll(a, "，", ".")
+	a = strings.ReplaceAll(a, "。", ".")
+	a = strings.ReplaceAll(a, "俛", "免")
+	eRegexp := regexp.MustCompile("[^\u4e00-\u9fa50-9a-zA-Z.。+]+")
+	effective := eRegexp.ReplaceAllString(a, "")
+
+	ra := []rune(effective)
+	final := ""
+
+	args := pinyin.NewArgs()
+	args.Fallback = func(r rune, a pinyin.Args) []string {
+		return []string{string(r)}
+	}
+
+	pys := pinyin.LazyConvert(effective, &args)
+	numMap := make(map[int]bool)
+	var diffNumCount int
+	var haveJia bool
+	var haveWei bool
+	var yue bool
+	var pao bool
+	var wang bool
+	var mian bool
+	var fei bool
+	for i, py := range pys {
+		c := string(ra[i])
+		n := -1
+		switch py {
+		case "ling", "ning", "lin", "nin", "O", "0":
+			n = 0
+			if py == "0" || py == "o" || py == "O" {
+				c = "o"
+			} else {
+				c = "0"
+			}
+		case "yi", "1":
+			n = 1
+			if py == "yi" {
+				c = "e"
+			}
+		case "er", "2":
+			n = 2
+			c = "2"
+		case "san", "sa", "shan", "3":
+			n = 3
+			c = "3"
+		case "si", "shi", "4":
+			n = 4
+			c = "4"
+		case "wu", "5":
+			n = 5
+			c = "5"
+		case "liu", "lu", "niu", "nu", "6":
+			n = 6
+			c = "6"
+		case "qi", "7":
+			n = 7
+			c = "7"
+		case "ba", "pa", "8":
+			n = 8
+			c = "8"
+		case "jiu", "9":
+			n = 9
+			c = "9"
+		case "dian":
+			c = "."
+		// "xi" 是常见汉字读音，不转成 "c"，避免“嘻嘻”等正常内容被误判为 .cc 域名。
+		case "se":
+			c = "c"
+		case "ti":
+			c = "t"
+		case "en", "eng":
+			c = "n"
+		case "ou":
+			c = "o"
+		case "pi":
+			c = "p"
+		case "jia", "+":
+			haveJia = true
+		case "wei", "v", "V", "w", "W", "qun", "qiong", "kou", "qiu", "q", "Q":
+			haveWei = true
+		case "wang":
+			wang = true
+		case "yue", "yo":
+			yue = true
+		case "pao", "bao":
+			pao = true
+		case "mian":
+			mian = true
+		case "fei":
+			fei = true
+		}
+
+		if n >= 0 {
+			if _, ok := numMap[n]; !ok {
+				numMap[n] = true
+				diffNumCount++
+			}
+		}
+
+		final += c
+	}
+
+	lRegexp := regexp.MustCompile(`[^a-zA-Z0-57-9]+`)
+	letter := lRegexp.ReplaceAllString(final, "")
+	//fmt.Println(effective)
+	//fmt.Println(final)
+	//fmt.Println(alpha)
+	// 数字检查
+	if diffNumCount >= 4 || len(letter) >= 5 {
+		return false
+	}
+	if haveJia && haveWei {
+		return false
+	}
+
+	alphaRegexp := regexp.MustCompile(`[^a-zA-Z0-9.]+`)
+	alpha := alphaRegexp.ReplaceAllString(final, "")
+	// 网址检查
+	if r := regexp.MustCompile(`(?i)\w*(aaa|abb|abc|ac|aco|ad|ads|ae|aeg|af|afl|ag|ai|aig|al|am|anz|ao|aol|app|aq|ar|art|as|at|au|aw|aws|ax|axa|az|ba|bar|bb|bbc|bbt|bcg|bcn|bd|be|bet|bf|bg|bh|bi|bid|bio|biz|bj|bm|bms|bmw|bn|bo|bom|boo|bot|box|br|bs|bt|buy|bv|bw|by|bz|bzh|ca|cab|cal|cam|car|cat|cba|cbn|cc|cd|ceo|cf|cfa|cfd|cg|ch|ci|ck|cl|cm|cn|co|com|cpa|cr|crs|cu|cv|cw|cx|cy|cz|dad|day|dds|de|dev|dhl|diy|dj|dk|dm|dnp|do|dog|dot|dtv|dvr|dz|eat|ec|eco|edu|ee|eg|er|es|esq|et|eu|eus|fan|fi|fit|fj|fk|fly|fm|fo|foo|fox|fr|frl|ftr|fun|fyi|hbo|hiv|hk|hkt|hm|hn|hot|how|hr|ht|hu|ibm|ice|icu|id|ie|ifm|il|im|in|inc|ing|ink|int|io|iq|ir|is|ist|it|itv|jcb|je|jio|jll|jm|jmp|jnj|jo|jot|joy|jp|ke|kfh|kg|kh|ki|kia|kim|km|kn|kp|kpn|kr|krd|kw|ky|kz|la|lat|law|lb|lc|lds|li|lk|llc|llp|lol|lpl|lr|ls|lt|ltd|lu|lv|ly|ma|man|map|mba|mc|md|me|med|men|mg|mh|mil|mit|mk|ml|mlb|mls|mm|mma|mn|mo|moe|moi|mom|mov|mp|mq|mr|ms|msd|mt|mtn|mtr|mu|mv|mw|mx|my|mz|na|nab|nba|nc|ne|nec|net|new|nf|nfl|ng|ngo|nhk|ni|nl|no|now|np|nr|nra|nrw|ntt|nu|nyc|nz|obi|om|one|ong|onl|ooo|org|ott|ovh|pa|pay|pe|pet|pf|pg|ph|phd|pid|pin|pk|pl|pm|pn|pnc|pr|pro|pru|ps|pt|pub|pw|pwc|py|qa|re|red|ren|ril|rio|rip|ro|rs|ru|run|rw|rwe|sa|sap|sas|sb|sbi|sbs|sc|scb|sd|se|sew|sex|sfr|sg|sh|si|sj|sk|ski|sky|sl|sm|sn|so|soy|spa|sr|srl|ss|st|stc|su|sv|sx|sy|sz|tab|tax|tc|tci|td|tdk|tel|tf|tg|th|thd|tj|tjx|tk|tl|tm|tn|to|top|tr|trv|tt|tui|tv|tvs|tw|tz|ua|ubs|ug|uk|uno|uol|ups|us|uy|uz|va|vc|ve|vet|vg|vi|vig|vin|vip|vn|vu|wed|wf|win|wme|wow|ws|wtc|wtf|xin|xxx|xyz|ye|you|yt|yun|za|zip|zm|zw)`); r != nil {
+		// 匹配成功
+		if r.MatchString(alpha) {
+			return false
+		}
+	}
+	if r := regexp.MustCompile(`(?i)\w+\.[0-1a-z]{2,}`); r != nil {
+		// 匹配成功
+		if r.MatchString(alpha) {
+			return false
+		}
+	}
+
+	// 综合检查
+	if (wang || yue || pao || (mian && fei)) && len(letter) >= 3 {
+		return false
+	}
+
+	return true
+}
+
+// checkRepeat 30分钟内超过5条一样的 拉入12小时黑名单
+func checkRepeat(a string) bool {
+	numRegexp := regexp.MustCompile("[^\u4e00-\u9fa5a-zA-Z]+")
+	a = numRegexp.ReplaceAllString(a, "")
+
+	var mLimit, cLimit = 30, 5
+	var black = 12
+	var hasHan bool
+	//var hasAlpha bool
+	rmap := make(map[int32]bool)
+	var rc int
+	for _, r := range a {
+		if unicode.Is(unicode.Han, r) {
+			hasHan = true
+		}
+		//if unicode.IsDigit(r) || isEnLetter(r) {
+		//	hasAlpha = true
+		//}
+		if _, ok := rmap[r]; !ok {
+			rmap[r] = true
+			rc++
+		}
+	}
+
+	// 必须包含汉字并且字母或数字 并且5个以上不重复字符
+	if !hasHan || rc < 5 {
+		return true
+	}
+
+	// 记录评论map
+	r := []rune(a)
+	if len(r) > 30 {
+		s1 := r[:30]
+		a = string(s1)
+	}
+
+	bKey := "cmt-b-" + a
+	key := "cmt-" + a
+	// 查看黑名单
+	_, ok := localcache.C.Get(bKey)
+	if ok {
+		return false
+	}
+	// 记录次数
+	c, ok2 := localcache.C.Get(key)
+	if !ok2 {
+		localcache.C.Set(key, 1, time.Duration(mLimit)*time.Minute)
+	} else {
+		// 超过次数加入黑名单
+		ci := c.(int)
+		if ci+1 >= cLimit {
+			localcache.C.Set(bKey, true, time.Duration(black)*time.Hour)
+			localcache.C.Delete(key)
+			return false
+		}
+		_ = localcache.C.Increment(key, 1)
+	}
+
+	return true
+}
+
+// 判断英文字母
+func isEnLetter(s rune) bool {
+	return (s >= 97 && s <= 122) || (s >= 65 && s <= 90)
+}
+
+type CommentValidator struct {
+	pattern *regexp.Regexp
+}
+
+func NewCommentValidator(pattern string) (*CommentValidator, error) {
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, err
+	}
+	return &CommentValidator{pattern: re}, nil
+}
+
+func (cv *CommentValidator) Validate(comment string) bool {
+	return cv.pattern.MatchString(comment)
+}

common/truthutil/truthutil_test.go

@@ -0,0 +1,38 @@
+package truthutil
+
+import "testing"
+
+func TestCheckURIAndNumberAllowsXiGreeting(t *testing.T) {
+	t.Parallel()
+
+	for _, content := range []string{
+		"嘻嘻",
+		"嘻嘻，我在这里",
+		"西西",
+	} {
+		content := content
+		t.Run(content, func(t *testing.T) {
+			t.Parallel()
+			if !checkURIAndNumber(content) {
+				t.Fatalf("checkURIAndNumber(%q) = false, want true", content)
+			}
+		})
+	}
+}
+
+func TestCheckURIAndNumberStillRejectsURLs(t *testing.T) {
+	t.Parallel()
+
+	for _, content := range []string{
+		"abc.com",
+		"abc点com",
+	} {
+		content := content
+		t.Run(content, func(t *testing.T) {
+			t.Parallel()
+			if checkURIAndNumber(content) {
+				t.Fatalf("checkURIAndNumber(%q) = true, want false", content)
+			}
+		})
+	}
+}