root/huangguo_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source 
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
root
2026-09-15 13:57:10 +08:00
co-authored by Claude Opus 5
commit 8679200f41
1897 changed files with 257900 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skd/service/contentreviewser/acg.go
+68
View File
@@ -0,0 +1,68 @@
package contentreviewser
import (
"91porn-server/common/log"
"91porn-server/models/v/contentreviewmod"
"91porn-server/models/v/mediacontentmod"
"91porn-server/models/v/mediamod"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
// acgScanner ACG 扫描策略：每个 Media 可能产生 1 个 ACG_MEDIA + N 个 ACG_CONTENT 命中
// 仅匹配 title / summary / 章节名 / 章节内容（不检测 tag）
type acgScanner struct{}
func (s *acgScanner) CountTotal() (int64, error) {
return mediamod.CountForReview()
}
func (s *acgScanner) ScanBatch(taskID, lastID primitive.ObjectID, terms []string) (*ScanResult, error) {
batch, err := mediamod.FindForReviewBatch(lastID, scanBatchSize)
if err != nil {
return nil, err
}
if len(batch) == 0 {
return &ScanResult{Done: true}, nil
}
issues := make([]*contentreviewmod.ReviewIssue, 0)
for _, m := range batch {
issues = append(issues, collectMediaIssues(taskID, m, terms)...)
}
return &ScanResult{
Issues: issues,
Checked: int64(len(batch)),
NextID: batch[len(batch)-1].ID,
}, nil
}
// collectMediaIssues 检测一个 Media 自身及其全部 MediaContent
func collectMediaIssues(taskID primitive.ObjectID, m *mediamod.Media, terms []string) []*contentreviewmod.ReviewIssue {
out := make([]*contentreviewmod.ReviewIssue, 0)
mb := newIssueBuilder(taskID, contentreviewmod.IssueTargetACGMedia, m.ID).
MatchTitle("标题", m.Title, terms).
MatchContent("内容", m.Summary, terms)
if mb.HasHits() {
out = append(out, mb.Build())
}
contents, err := mediacontentmod.QueryAllList(bson.M{"mediaId": m.ID, "isActive": true})
if err != nil {
log.Warn("query media contents fail", log.Any("mid", m.ID), log.E(err))
return out
}
for _, c := range contents {
cb := newIssueBuilder(taskID, contentreviewmod.IssueTargetACGContent, c.ID).
WithMediaID(m.ID).
MatchTitle("章节名", c.Name, terms).
MatchContent("章节内容", c.Text, terms)
if cb.HasHits() {
out = append(out, cb.Build())
}
}
return out
}
skd/service/contentreviewser/issue_builder.go
+92
View File
@@ -0,0 +1,92 @@
package contentreviewser
import (
"fmt"
"strings"
"91porn-server/models/v/contentreviewmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
// issueBuilder 用 Builder 模式构建 ReviewIssue：
// 通过链式调用 MatchXxx 收集各字段的命中，HasHits/Build 在所有匹配后调用
type issueBuilder struct {
issue *contentreviewmod.ReviewIssue
hitSet map[string]struct{}
descLines []string
}
func newIssueBuilder(taskID primitive.ObjectID, targetType contentreviewmod.IssueTargetType, targetID primitive.ObjectID) *issueBuilder {
return &issueBuilder{
issue: &contentreviewmod.ReviewIssue{
TaskID: taskID,
TargetType: targetType,
TargetID: targetID,
},
hitSet: make(map[string]struct{}),
}
}
// WithMediaID 设置关联的 MediaID（仅 ACG_CONTENT 需要）
func (b *issueBuilder) WithMediaID(id primitive.ObjectID) *issueBuilder {
b.issue.MediaID = id
return b
}
// MatchTitle 匹配标题字段；命中时存入高亮版本并累计描述
// label 是描述文案中的字段名（如"标题"、"章节名"）
func (b *issueBuilder) MatchTitle(label, text string, terms []string) *issueBuilder {
if hits, hl := matchAndHighlight(text, terms); len(hits) > 0 {
b.issue.Title = hl
b.addHits(label, hits)
}
return b
}
func (b *issueBuilder) MatchContent(label, text string, terms []string) *issueBuilder {
if hits, hl := matchAndHighlight(text, terms); len(hits) > 0 {
b.issue.Content = hl
b.addHits(label, hits)
}
return b
}
func (b *issueBuilder) MatchTags(label, text string, terms []string) *issueBuilder {
if hits, hl := matchAndHighlight(text, terms); len(hits) > 0 {
b.issue.Tags = hl
b.addHits(label, hits)
}
return b
}
func (b *issueBuilder) MatchRichText(label, text string, terms []string) *issueBuilder {
if hits, hl := matchAndHighlight(text, terms); len(hits) > 0 {
b.issue.RichText = hl
b.addHits(label, hits)
}
return b
}
// HasHits 是否至少有一个字段命中
func (b *issueBuilder) HasHits() bool {
return len(b.hitSet) > 0
}
// Build 收尾：拼装 Description 与去重后的 HitWords
func (b *issueBuilder) Build() *contentreviewmod.ReviewIssue {
hits := make([]string, 0, len(b.hitSet))
for h := range b.hitSet {
hits = append(hits, h)
}
b.issue.HitWords = hits
b.issue.Description = strings.Join(b.descLines, "\n")
return b.issue
}
func (b *issueBuilder) addHits(label string, hits []string) {
b.descLines = append(b.descLines, fmt.Sprintf("%s命中 %s", label, strings.Join(hits, "、")))
for _, h := range hits {
b.hitSet[h] = struct{}{}
}
}
skd/service/contentreviewser/matcher.go
+114
View File
@@ -0,0 +1,114 @@
package contentreviewser
import (
"sort"
"strings"
"91porn-server/models/v/contentreviewmod"
)
const (
markPrefix = "<mark>"
markSuffix = "</mark>"
)
// allTerms 把快照展开为一组待匹配的词
func allTerms(snaps []contentreviewmod.SensitiveWordSnap) []string {
seen := make(map[string]struct{})
out := make([]string, 0, len(snaps))
for _, s := range snaps {
if s.Word == "" {
continue
}
if _, ok := seen[s.Word]; ok {
continue
}
seen[s.Word] = struct{}{}
out = append(out, s.Word)
}
// 长词优先，避免短词先替换破坏长词的匹配
sort.Slice(out, func(i, j int) bool {
return len(out[i]) > len(out[j])
})
return out
}
// findHits 返回 input 中命中的词（去重，保持长→短顺序）
func findHits(input string, terms []string) []string {
if input == "" || len(terms) == 0 {
return nil
}
hits := make([]string, 0)
seen := make(map[string]struct{})
for _, t := range terms {
if t == "" {
continue
}
if _, ok := seen[t]; ok {
continue
}
if strings.Contains(input, t) {
seen[t] = struct{}{}
hits = append(hits, t)
}
}
return hits
}
// highlight 用 <mark>...</mark> 包裹命中的词
// hits 已按长度降序；为避免再次包裹已 mark 的内容，分段替换
func highlight(input string, hits []string) string {
if input == "" || len(hits) == 0 {
return input
}
result := input
for _, h := range hits {
if h == "" {
continue
}
result = replaceNotInMark(result, h)
}
return result
}
// replaceNotInMark 在 input 中把 term 全部替换为 <mark>term</mark>，
// 但跳过已经包在 <mark>...</mark> 内的片段（避免嵌套）
func replaceNotInMark(input, term string) string {
if term == "" {
return input
}
var b strings.Builder
i := 0
for i < len(input) {
// 已经在 mark 块里，整段原样输出
if strings.HasPrefix(input[i:], markPrefix) {
end := strings.Index(input[i:], markSuffix)
if end < 0 {
b.WriteString(input[i:])
return b.String()
}
b.WriteString(input[i : i+end+len(markSuffix)])
i += end + len(markSuffix)
continue
}
if strings.HasPrefix(input[i:], term) {
b.WriteString(markPrefix)
b.WriteString(term)
b.WriteString(markSuffix)
i += len(term)
continue
}
b.WriteByte(input[i])
i++
}
return b.String()
}
// matchAndHighlight 在 input 上一步到位：返回命中词列表 + 高亮后的文本
func matchAndHighlight(input string, terms []string) ([]string, string) {
hits := findHits(input, terms)
if len(hits) == 0 {
return nil, input
}
return hits, highlight(input, hits)
}
skd/service/contentreviewser/runner.go
+129
View File
@@ -0,0 +1,129 @@
package contentreviewser
import (
"fmt"
"runtime/debug"
"sync/atomic"
"91porn-server/common/log"
"91porn-server/models/v/contentreviewmod"
"91porn-server/models/v/sensitivewordmod"
"91porn-server/skd/service/export_task"
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
)
// tgChatID 内容审查任务完成通知群（与导出任务复用同一群）
const tgChatID int64 = -1003399433452
var runningFlag int32 // 进程内互斥：0=空闲 1=运行中
// TryRunPendingTask 调度入口，由 skd 的 cron 定时调用
// skd 单进程，仅需 atomic 互斥防止 cron tick 重入；每次只挑一个最早的任务处理
// 取到 pending 视为首跑；取到心跳超时的 running 视为续跑，从 lastTargetId 继续
func TryRunPendingTask() {
if !atomic.CompareAndSwapInt32(&runningFlag, 0, 1) {
return
}
defer atomic.StoreInt32(&runningFlag, 0)
task, err := contentreviewmod.PickNextTask()
if err != nil {
log.Warn("content review: pick next fail", log.E(err))
return
}
if task == nil {
return
}
if task.ID.IsZero() {
return
}
isResume := task.Status == contentreviewmod.TaskStatusRunning
if err := contentreviewmod.AcquireTask(task.ID, isResume); err != nil {
log.Warn("content review: acquire task fail", log.Any("taskId", task.ID), log.E(err))
return
}
if isResume {
log.Info("content review: resume stale task",
log.Any("taskId", task.ID), log.Any("lastTargetId", task.LastTargetID))
}
executeTask(task)
}
// executeTask 真正执行扫描
func executeTask(task *contentreviewmod.ReviewTask) {
defer func() {
if r := recover(); r != nil {
stack := string(debug.Stack())
log.Error("content review task panic",
log.Any("taskId", task.ID), log.Any("panic", r), log.Any("stack", stack))
_ = contentreviewmod.MarkFailed(task.ID, fmt.Sprintf("panic: %v", r))
}
}()
log.Info("content review task start",
log.Any("taskId", task.ID), log.Any("type", task.TaskType))
checked, issues, err := runScan(task)
if err != nil {
log.Error("content review task fail",
log.Any("taskId", task.ID), log.E(err))
_ = contentreviewmod.MarkFailed(task.ID, err.Error())
return
}
if err := contentreviewmod.MarkFinished(task.ID, checked, issues); err != nil {
log.Error("content review mark finished fail", log.E(err))
return
}
// 发送 TG
if sendTGNotify(task, checked, issues) {
_ = contentreviewmod.MarkTgSent(task.ID)
}
log.Info("content review task done",
log.Any("taskId", task.ID), log.Any("checked", checked), log.Any("issues", issues))
}
func sendTGNotify(task *contentreviewmod.ReviewTask, checked, issues int64) bool {
if export_task.Bot == nil {
log.Warn("content review: tg bot not initialized, skip notify")
return false
}
typeName := "视频帖子"
if task.TaskType == contentreviewmod.TaskTypeACG {
typeName = "ACG"
}
operator := task.Operator
if operator == "" {
operator = "-"
}
text := fmt.Sprintf(
"【%s 内容检测任务完成】\n操作人: %s\n类型: %s\n任务ID: %s\n总记录: %d\n已检测: %d\n命中: %d",
export_task.GetProName(), operator, typeName, task.ID.Hex(), task.TotalCount, checked, issues,
)
if _, err := export_task.Bot.Send(tgbotapi.NewMessage(tgChatID, text)); err != nil {
log.Warn("content review: send tg fail", log.Any("taskId", task.ID), log.E(err))
return false
}
return true
}
// BuildSnapshot 拉取当前启用的敏感词作为快照（web 创建任务时复用）
func BuildSnapshot() ([]contentreviewmod.SensitiveWordSnap, error) {
enabled := sensitivewordmod.StatusEnabled
list, err := sensitivewordmod.FindAll(&sensitivewordmod.ListReq{Status: &enabled})
if err != nil {
return nil, err
}
out := make([]contentreviewmod.SensitiveWordSnap, 0, len(list))
for _, w := range list {
out = append(out, contentreviewmod.SensitiveWordSnap{
Word: w.Word,
Category: w.Category,
})
}
return out, nil
}
skd/service/contentreviewser/scanner.go
+95
View File
@@ -0,0 +1,95 @@
package contentreviewser
import (
"fmt"
"91porn-server/common/log"
"91porn-server/models/v/contentreviewmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
const scanBatchSize = 100
// ScanResult 一批扫描的产出
type ScanResult struct {
Issues []*contentreviewmod.ReviewIssue // 本批生成的命中记录
Checked int64 // 本批实际扫描的目标数量
NextID primitive.ObjectID // 下一批的游标
Done bool // true 表示已扫完，runScan 退出循环
}
// Scanner 内容审查扫描策略；具体任务类型实现各自的 ScanBatch
type Scanner interface {
CountTotal() (int64, error)
ScanBatch(taskID, lastID primitive.ObjectID, terms []string) (*ScanResult, error)
}
// newScanner Strategy 工厂：根据任务类型返回具体扫描器
func newScanner(taskType string) (Scanner, error) {
switch taskType {
case contentreviewmod.TaskTypeVideo:
return &videoScanner{}, nil
case contentreviewmod.TaskTypeACG:
return &acgScanner{}, nil
default:
return nil, fmt.Errorf("unsupported task type: %s", taskType)
}
}
// runScan Template Method：把"分页游标 + 进度回写 + 批量插入"骨架抽出，差异由 Scanner 提供
// 支持从 task.LastTargetID 断点续跑
func runScan(task *contentreviewmod.ReviewTask) (int64, int64, error) {
scanner, err := newScanner(task.TaskType)
if err != nil {
return task.CheckedCount, task.IssueCount, err
}
terms := allTerms(task.SnapshotWords)
total := task.TotalCount
if total <= 0 {
t, err := scanner.CountTotal()
if err != nil {
return task.CheckedCount, task.IssueCount, fmt.Errorf("count total: %w", err)
}
total = t
}
// 同步到内存对象，便于 executeTask 拼 TG 消息时正确取值
task.TotalCount = total
lastID := task.LastTargetID
checked := task.CheckedCount
issues := task.IssueCount
flushProgress(task.ID, lastID, checked, issues, total)
for {
result, err := scanner.ScanBatch(task.ID, lastID, terms)
if err != nil {
return checked, issues, fmt.Errorf("scan batch: %w", err)
}
if result.Done {
break
}
if len(result.Issues) > 0 {
if err := contentreviewmod.InsertIssues(result.Issues); err != nil {
log.Warn("insert issues fail", log.Any("count", len(result.Issues)), log.E(err))
} else {
issues += int64(len(result.Issues))
}
}
checked += result.Checked
lastID = result.NextID
flushProgress(task.ID, lastID, checked, issues, total)
}
return checked, issues, nil
}
func flushProgress(id, lastID primitive.ObjectID, checked, issues, total int64) {
if err := contentreviewmod.UpdateProgress(id, lastID, checked, issues, total); err != nil {
log.Warn("update progress fail", log.E(err))
}
}
skd/service/contentreviewser/video.go
+42
View File
@@ -0,0 +1,42 @@
package contentreviewser
import (
"91porn-server/models/v/contentreviewmod"
"91porn-server/models/v/vidmod"
"go.mongodb.org/mongo-driver/bson/primitive"
)
// videoScanner 视频帖子扫描策略；匹配 title / content / richText（不检测 tag）
type videoScanner struct{}
func (s *videoScanner) CountTotal() (int64, error) {
return vidmod.CountForReview()
}
func (s *videoScanner) ScanBatch(taskID, lastID primitive.ObjectID, terms []string) (*ScanResult, error) {
batch, err := vidmod.FindForReviewBatch(lastID, scanBatchSize)
if err != nil {
return nil, err
}
if len(batch) == 0 {
return &ScanResult{Done: true}, nil
}
issues := make([]*contentreviewmod.ReviewIssue, 0, len(batch))
for _, v := range batch {
b := newIssueBuilder(taskID, contentreviewmod.IssueTargetVideo, v.ID).
MatchTitle("标题", v.Title, terms).
MatchContent("内容", v.Content, terms).
MatchRichText("富文本", v.RichText, terms)
if b.HasHits() {
issues = append(issues, b.Build())
}
}
return &ScanResult{
Issues: issues,
Checked: int64(len(batch)),
NextID: batch[len(batch)-1].ID,
}, nil
}