web_hghl_server/app/skd/job/llmstxt/llmstxt.go

package llmstxt

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
	"videoweb/app/service/configser"
	"videoweb/pub/common/log"
	"videoweb/pub/common/utils"
	"videoweb/pub/models/actress"
	"videoweb/pub/models/videocategorymod"
	"videoweb/pub/models/videoinformationmod"
	"videoweb/pub/models/videotagmod"
	"videoweb/pub/pubdef"

	"go.mongodb.org/mongo-driver/bson"
	"go.mongodb.org/mongo-driver/mongo/options"
)

const (
	maxTagsInBrief  = 80  // llms.txt 中标签条数上限
	maxRecentInFull = 500 // llms-full.txt 中最新视频条数上限
)

var generateMu sync.Mutex

// GenerateLLMSTxt 生成 llms.txt（摘要）和 llms-full.txt（详细），输出到 outDir。
func GenerateLLMSTxt(outDir string) {
	generateMu.Lock()
	defer generateMu.Unlock()

	ctx := context.Background()
	config, serr := configser.GetSystemConfig(ctx)
	if serr != nil || config == nil {
		log.Error("GenerateLLMSTxt GetSystemConfig", log.Any("err", serr))
		return
	}

	baseURL := strings.TrimSuffix(config.WebAddr, "/")

	// ── 查询数据 ──────────────────────────────────────────────
	categoryList, _ := videocategorymod.FindManyVideoCategorys(ctx,
		bson.M{videocategorymod.FN.VideoCategory.Status: true},
		func() *options.FindOptions {
			o := options.Find()
			o.SetSort(bson.M{videocategorymod.FN.VideoCategory.Rank: -1})
			return o
		}())

	tagOpt := options.Find()
	tagOpt.SetSort(bson.M{videotagmod.FN.VideoTagInfo.WatchTimes: -1})
	tagOpt.SetLimit(int64(maxTagsInBrief))
	tagList, _ := videotagmod.FindManyVideoTagInfos(ctx,
		bson.M{videotagmod.FN.VideoTagInfo.Status: true}, tagOpt)

	actressOpt := options.Find()
	actressOpt.SetSort(bson.M{actress.FN.Actress.Works: -1})
	actressList, _ := actress.FindManyActresss(ctx,
		bson.M{actress.FN.Actress.Status: true}, actressOpt)

	recentOpt := options.Find()
	recentOpt.SetSort(bson.M{videoinformationmod.FN.VideoInformation.AddedTime: -1})
	recentOpt.SetLimit(50)
	recentVideos, _ := videoinformationmod.FindManyVideoInformations(ctx,
		bson.M{videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK},
		recentOpt)

	// ── 生成 llms.txt ─────────────────────────────────────────
	var brief strings.Builder
	writeBrief(&brief, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld,
		categoryList, tagList, actressList, recentVideos)

	if err := writeTextFile(outDir, "llms.txt", brief.String()); err != nil {
		log.Error("GenerateLLMSTxt write llms.txt", log.Any("err", err))
		return
	}
	if err := commitFile(outDir, "llms.txt"); err != nil {
		log.Error("GenerateLLMSTxt commit llms.txt", log.Any("err", err))
		return
	}

	// ── 生成 llms-full.txt ────────────────────────────────────
	allVideoOpt := options.Find()
	allVideoOpt.SetSort(bson.M{videoinformationmod.FN.VideoInformation.AddedTime: -1})
	allVideoOpt.SetLimit(int64(maxRecentInFull))
	allVideos, _ := videoinformationmod.FindManyVideoInformations(ctx,
		bson.M{videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK},
		allVideoOpt)

	var full strings.Builder
	writeFull(&full, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld,
		categoryList, tagList, actressList, allVideos)

	if err := writeTextFile(outDir, "llms-full.txt", full.String()); err != nil {
		log.Error("GenerateLLMSTxt write llms-full.txt", log.Any("err", err))
		return
	}
	if err := commitFile(outDir, "llms-full.txt"); err != nil {
		log.Error("GenerateLLMSTxt commit llms-full.txt", log.Any("err", err))
		return
	}

	log.Info("GenerateLLMSTxt done",
		log.Any("categories", len(categoryList)),
		log.Any("tags", len(tagList)),
		log.Any("actresses", len(actressList)),
		log.Any("videos", len(allVideos)),
	)
}

// ── 内容构建 ──────────────────────────────────────────────────

func writeBrief(
	b *strings.Builder,
	title, desc, baseURL string,
	keywords []string,
	categories videocategorymod.VideoCategoryList,
	tags videotagmod.VideoTagInfoList,
	actresses actress.ActressList,
	recentVideos videoinformationmod.VideoInformationList,
) {
	now := time.Now().Format("2006-01-02")

	// 文件头
	fmt.Fprintf(b, "# %s\n\n", title)
	if desc != "" {
		fmt.Fprintf(b, "> %s\n\n", desc)
	}
	if len(keywords) > 0 {
		fmt.Fprintf(b, "> 关键词：%s\n\n", strings.Join(keywords, "、"))
	}
	fmt.Fprintf(b, "> 更新时间：%s\n\n", now)

	// 重要页面
	writeImportantPages(b, baseURL)

	// 分类
	if len(categories) > 0 {
		fmt.Fprintf(b, "## 内容分类\n\n")
		for _, c := range categories {
			if c == nil || c.Id == 0 {
				continue
			}
			name := c.Name
			if c.Title != "" {
				name = c.Title
			}
			catDesc := c.Desc
			if catDesc == "" {
				catDesc = name + "相关视频内容"
			}
			fmt.Fprintf(b, "- [%s](%s/video/more/%d): %s\n", name,
				baseURL, c.Id, catDesc)
		}
		b.WriteString("\n")
	}

	// 热门标签（取播放量最高的）
	if len(tags) > 0 {
		fmt.Fprintf(b, "## 热门标签\n\n")
		for _, t := range tags {
			if t == nil || t.Id == 0 {
				continue
			}
			fmt.Fprintf(b, "- [%s](%s/video/all?pageNum=1&pageSize=40&tagsIds=%d)\n", t.Title, baseURL, t.Id)
		}
		b.WriteString("\n")
	}

	// 作者
	if len(actresses) > 0 {
		fmt.Fprintf(b, "## 作者\n\n")
		for _, item := range actresses {
			if item == nil || item.Id == 0 {
				continue
			}
			fmt.Fprintf(b, "- [%s](%s/actress/detail?id=%d)\n", item.Alias, baseURL, item.Id)
		}
		b.WriteString("\n")
	}

	// 最新内容（50条）
	if len(recentVideos) > 0 {
		fmt.Fprintf(b, "## 最新内容\n\n")
		for _, v := range recentVideos {
			if v == nil || v.Id == 0 {
				continue
			}
			url := utils.JoinUrlPath(baseURL, fmt.Sprintf("/video/detail?id=%d", v.Id))
			desc := v.Desc
			if desc == "" {
				desc = v.Title
			}
			desc = truncate(desc, 100)
			fmt.Fprintf(b, "- [%s](%s): %s\n", v.Title, url, desc)
		}
		b.WriteString("\n")
	}
}

func writeFull(
	b *strings.Builder,
	title, desc, baseURL string,
	keywords []string,
	categories videocategorymod.VideoCategoryList,
	tags videotagmod.VideoTagInfoList,
	actresses actress.ActressList,
	videos videoinformationmod.VideoInformationList,
) {
	// llms-full.txt 先包含 brief 的所有内容（用全量视频列表替换最新50条）
	// 再附加每条视频的完整详情

	now := time.Now().Format("2006-01-02")

	fmt.Fprintf(b, "# %s\n\n", title)
	if desc != "" {
		fmt.Fprintf(b, "> %s\n\n", desc)
	}
	if len(keywords) > 0 {
		fmt.Fprintf(b, "> 关键词：%s\n\n", strings.Join(keywords, "、"))
	}
	fmt.Fprintf(b, "> 更新时间：%s\n\n", now)

	// 重要页面
	writeImportantPages(b, baseURL)

	// 分类（含 SEO 描述）
	if len(categories) > 0 {
		fmt.Fprintf(b, "## 内容分类\n\n")
		for _, c := range categories {
			if c == nil || c.Id == 0 {
				continue
			}
			name := c.Name
			if c.Title != "" {
				name = c.Title
			}
			catDesc := c.SeoInfo.Description
			if catDesc == "" {
				catDesc = c.Desc
			}
			if catDesc == "" {
				catDesc = name + "相关视频内容"
			}
			fmt.Fprintf(b, "- [%s](%s/video/more/%d): %s\n", name, baseURL, c.Id, catDesc)
		}
		b.WriteString("\n")
	}

	// 全部标签
	if len(tags) > 0 {
		fmt.Fprintf(b, "## 全部标签\n\n")
		for _, t := range tags {
			if t == nil || t.Id == 0 {
				continue
			}
			tagDesc := t.SeoInfo.Description
			if tagDesc == "" {
				tagDesc = t.Title
			}
			fmt.Fprintf(b, "- [%s](%s/video/all?pageNum=1&pageSize=40&tagsIds=%d): %s\n", t.Title, baseURL, t.Id, tagDesc)
		}
		b.WriteString("\n")
	}

	// 作者
	if len(actresses) > 0 {
		fmt.Fprintf(b, "## 作者\n\n")
		for _, item := range actresses {
			if item == nil || item.Id == 0 {
				continue
			}
			itemDesc := item.SeoInfo.Description
			if itemDesc == "" {
				itemDesc = item.Desc
			}
			fmt.Fprintf(b, "- [%s](%s/actress/detail?id=%d): %s\n",
				item.Alias, baseURL, item.Id, itemDesc)
		}
		b.WriteString("\n")
	}

	// 视频详情列表
	if len(videos) > 0 {
		fmt.Fprintf(b, "## 内容详情（最新 %d 条）\n\n", len(videos))
		for _, v := range videos {
			if v == nil || v.Id == 0 {
				continue
			}
			url := utils.JoinUrlPath(baseURL, fmt.Sprintf("/video/detail?id=%d", v.Id))
			fmt.Fprintf(b, "### [%s](%s)\n\n", v.Title, url)

			if v.Desc != "" {
				fmt.Fprintf(b, "%s\n\n", truncate(v.Desc, 500))
			}

			if !v.AddedTime.IsZero() {
				fmt.Fprintf(b, "- 发布时间：%s\n", v.AddedTime.Format("2006-01-02"))
			}
			if v.CreatorName != "" {
				fmt.Fprintf(b, "- 创作者：%s\n", v.CreatorName)
			}
			if len(v.Keyworld) > 0 {
				fmt.Fprintf(b, "- 关键词：%s\n", strings.Join(v.Keyworld, "、"))
			}
			b.WriteString("\n")
		}
	}
}

func writeImportantPages(b *strings.Builder, baseURL string) {
	fmt.Fprintf(b, "## 重要页面\n\n")
	fmt.Fprintf(b, "- [首页](%s/): 网站主页\n", baseURL)
	fmt.Fprintf(b, "- [热门搜索](%s/search/hot): 热门搜索关键词\n", baseURL)
	fmt.Fprintf(b, "- [视频首页](%s/video): 高清视频内容入口\n", baseURL)
	fmt.Fprintf(b, "- [全部视频](%s/video/all): 浏览全部视频\n", baseURL)
	fmt.Fprintf(b, "- [视频排行榜](%s/video/rank): 热门视频排行\n", baseURL)
	fmt.Fprintf(b, "- [线路导航](%s/nav): 最新可用线路入口\n", baseURL)
	fmt.Fprintf(b, "- [应用导航](%s/application): 应用推荐与下载入口\n", baseURL)
	fmt.Fprintf(b, "- [RSS订阅](%s/rss.xml): RSS Feed\n", baseURL)
	fmt.Fprintf(b, "- [Sitemap](%s/sitemap.xml): 网站地图\n\n", baseURL)
}

// ── 工具函数 ──────────────────────────────────────────────────

func truncate(s string, maxLen int) string {
	runes := []rune(s)
	if len(runes) <= maxLen {
		return s
	}
	return string(runes[:maxLen]) + "…"
}

func writeTextFile(outDir, filename, content string) error {
	if err := os.MkdirAll(outDir, 0755); err != nil {
		return fmt.Errorf("mkdir %s: %w", outDir, err)
	}
	tmpPath := filepath.Join(outDir, filename+".tmp")
	f, err := os.Create(tmpPath)
	if err != nil {
		return fmt.Errorf("create %s: %w", tmpPath, err)
	}
	defer f.Close()
	if _, err := f.WriteString(content); err != nil {
		return fmt.Errorf("write %s: %w", filename, err)
	}
	return f.Sync()
}

func commitFile(outDir, filename string) error {
	src := filepath.Join(outDir, filename+".tmp")
	dst := filepath.Join(outDir, filename)
	if err := os.Rename(src, dst); err != nil {
		os.Remove(src)
		return fmt.Errorf("rename %s: %w", filename, err)
	}
	return nil
}