Initial commit: web_hghl_server
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,362 @@
|
||||
package llmstxt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/app/service/configser"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/utils"
|
||||
"videoweb/pub/models/actress"
|
||||
"videoweb/pub/models/videocategorymod"
|
||||
"videoweb/pub/models/videoinformationmod"
|
||||
"videoweb/pub/models/videotagmod"
|
||||
"videoweb/pub/pubdef"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const (
|
||||
maxTagsInBrief = 80 // llms.txt 中标签条数上限
|
||||
maxRecentInFull = 500 // llms-full.txt 中最新视频条数上限
|
||||
)
|
||||
|
||||
var generateMu sync.Mutex
|
||||
|
||||
// GenerateLLMSTxt 生成 llms.txt(摘要)和 llms-full.txt(详细),输出到 outDir。
|
||||
func GenerateLLMSTxt(outDir string) {
|
||||
generateMu.Lock()
|
||||
defer generateMu.Unlock()
|
||||
|
||||
ctx := context.Background()
|
||||
config, serr := configser.GetSystemConfig(ctx)
|
||||
if serr != nil || config == nil {
|
||||
log.Error("GenerateLLMSTxt GetSystemConfig", log.Any("err", serr))
|
||||
return
|
||||
}
|
||||
|
||||
baseURL := strings.TrimSuffix(config.WebAddr, "/")
|
||||
|
||||
// ── 查询数据 ──────────────────────────────────────────────
|
||||
categoryList, _ := videocategorymod.FindManyVideoCategorys(ctx,
|
||||
bson.M{videocategorymod.FN.VideoCategory.Status: true},
|
||||
func() *options.FindOptions {
|
||||
o := options.Find()
|
||||
o.SetSort(bson.M{videocategorymod.FN.VideoCategory.Rank: -1})
|
||||
return o
|
||||
}())
|
||||
|
||||
tagOpt := options.Find()
|
||||
tagOpt.SetSort(bson.M{videotagmod.FN.VideoTagInfo.WatchTimes: -1})
|
||||
tagOpt.SetLimit(int64(maxTagsInBrief))
|
||||
tagList, _ := videotagmod.FindManyVideoTagInfos(ctx,
|
||||
bson.M{videotagmod.FN.VideoTagInfo.Status: true}, tagOpt)
|
||||
|
||||
actressOpt := options.Find()
|
||||
actressOpt.SetSort(bson.M{actress.FN.Actress.Works: -1})
|
||||
actressList, _ := actress.FindManyActresss(ctx,
|
||||
bson.M{actress.FN.Actress.Status: true}, actressOpt)
|
||||
|
||||
recentOpt := options.Find()
|
||||
recentOpt.SetSort(bson.M{videoinformationmod.FN.VideoInformation.AddedTime: -1})
|
||||
recentOpt.SetLimit(50)
|
||||
recentVideos, _ := videoinformationmod.FindManyVideoInformations(ctx,
|
||||
bson.M{videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK},
|
||||
recentOpt)
|
||||
|
||||
// ── 生成 llms.txt ─────────────────────────────────────────
|
||||
var brief strings.Builder
|
||||
writeBrief(&brief, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld,
|
||||
categoryList, tagList, actressList, recentVideos)
|
||||
|
||||
if err := writeTextFile(outDir, "llms.txt", brief.String()); err != nil {
|
||||
log.Error("GenerateLLMSTxt write llms.txt", log.Any("err", err))
|
||||
return
|
||||
}
|
||||
if err := commitFile(outDir, "llms.txt"); err != nil {
|
||||
log.Error("GenerateLLMSTxt commit llms.txt", log.Any("err", err))
|
||||
return
|
||||
}
|
||||
|
||||
// ── 生成 llms-full.txt ────────────────────────────────────
|
||||
allVideoOpt := options.Find()
|
||||
allVideoOpt.SetSort(bson.M{videoinformationmod.FN.VideoInformation.AddedTime: -1})
|
||||
allVideoOpt.SetLimit(int64(maxRecentInFull))
|
||||
allVideos, _ := videoinformationmod.FindManyVideoInformations(ctx,
|
||||
bson.M{videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK},
|
||||
allVideoOpt)
|
||||
|
||||
var full strings.Builder
|
||||
writeFull(&full, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld,
|
||||
categoryList, tagList, actressList, allVideos)
|
||||
|
||||
if err := writeTextFile(outDir, "llms-full.txt", full.String()); err != nil {
|
||||
log.Error("GenerateLLMSTxt write llms-full.txt", log.Any("err", err))
|
||||
return
|
||||
}
|
||||
if err := commitFile(outDir, "llms-full.txt"); err != nil {
|
||||
log.Error("GenerateLLMSTxt commit llms-full.txt", log.Any("err", err))
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("GenerateLLMSTxt done",
|
||||
log.Any("categories", len(categoryList)),
|
||||
log.Any("tags", len(tagList)),
|
||||
log.Any("actresses", len(actressList)),
|
||||
log.Any("videos", len(allVideos)),
|
||||
)
|
||||
}
|
||||
|
||||
// ── 内容构建 ──────────────────────────────────────────────────
|
||||
|
||||
func writeBrief(
|
||||
b *strings.Builder,
|
||||
title, desc, baseURL string,
|
||||
keywords []string,
|
||||
categories videocategorymod.VideoCategoryList,
|
||||
tags videotagmod.VideoTagInfoList,
|
||||
actresses actress.ActressList,
|
||||
recentVideos videoinformationmod.VideoInformationList,
|
||||
) {
|
||||
now := time.Now().Format("2006-01-02")
|
||||
|
||||
// 文件头
|
||||
fmt.Fprintf(b, "# %s\n\n", title)
|
||||
if desc != "" {
|
||||
fmt.Fprintf(b, "> %s\n\n", desc)
|
||||
}
|
||||
if len(keywords) > 0 {
|
||||
fmt.Fprintf(b, "> 关键词:%s\n\n", strings.Join(keywords, "、"))
|
||||
}
|
||||
fmt.Fprintf(b, "> 更新时间:%s\n\n", now)
|
||||
|
||||
// 重要页面
|
||||
writeImportantPages(b, baseURL)
|
||||
|
||||
// 分类
|
||||
if len(categories) > 0 {
|
||||
fmt.Fprintf(b, "## 内容分类\n\n")
|
||||
for _, c := range categories {
|
||||
if c == nil || c.Id == 0 {
|
||||
continue
|
||||
}
|
||||
name := c.Name
|
||||
if c.Title != "" {
|
||||
name = c.Title
|
||||
}
|
||||
catDesc := c.Desc
|
||||
if catDesc == "" {
|
||||
catDesc = name + "相关视频内容"
|
||||
}
|
||||
fmt.Fprintf(b, "- [%s](%s/video/more/%d): %s\n", name,
|
||||
baseURL, c.Id, catDesc)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// 热门标签(取播放量最高的)
|
||||
if len(tags) > 0 {
|
||||
fmt.Fprintf(b, "## 热门标签\n\n")
|
||||
for _, t := range tags {
|
||||
if t == nil || t.Id == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(b, "- [%s](%s/video/all?pageNum=1&pageSize=40&tagsIds=%d)\n", t.Title, baseURL, t.Id)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// 作者
|
||||
if len(actresses) > 0 {
|
||||
fmt.Fprintf(b, "## 作者\n\n")
|
||||
for _, item := range actresses {
|
||||
if item == nil || item.Id == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(b, "- [%s](%s/actress/detail?id=%d)\n", item.Alias, baseURL, item.Id)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// 最新内容(50条)
|
||||
if len(recentVideos) > 0 {
|
||||
fmt.Fprintf(b, "## 最新内容\n\n")
|
||||
for _, v := range recentVideos {
|
||||
if v == nil || v.Id == 0 {
|
||||
continue
|
||||
}
|
||||
url := utils.JoinUrlPath(baseURL, fmt.Sprintf("/video/detail?id=%d", v.Id))
|
||||
desc := v.Desc
|
||||
if desc == "" {
|
||||
desc = v.Title
|
||||
}
|
||||
desc = truncate(desc, 100)
|
||||
fmt.Fprintf(b, "- [%s](%s): %s\n", v.Title, url, desc)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
|
||||
func writeFull(
|
||||
b *strings.Builder,
|
||||
title, desc, baseURL string,
|
||||
keywords []string,
|
||||
categories videocategorymod.VideoCategoryList,
|
||||
tags videotagmod.VideoTagInfoList,
|
||||
actresses actress.ActressList,
|
||||
videos videoinformationmod.VideoInformationList,
|
||||
) {
|
||||
// llms-full.txt 先包含 brief 的所有内容(用全量视频列表替换最新50条)
|
||||
// 再附加每条视频的完整详情
|
||||
|
||||
now := time.Now().Format("2006-01-02")
|
||||
|
||||
fmt.Fprintf(b, "# %s\n\n", title)
|
||||
if desc != "" {
|
||||
fmt.Fprintf(b, "> %s\n\n", desc)
|
||||
}
|
||||
if len(keywords) > 0 {
|
||||
fmt.Fprintf(b, "> 关键词:%s\n\n", strings.Join(keywords, "、"))
|
||||
}
|
||||
fmt.Fprintf(b, "> 更新时间:%s\n\n", now)
|
||||
|
||||
// 重要页面
|
||||
writeImportantPages(b, baseURL)
|
||||
|
||||
// 分类(含 SEO 描述)
|
||||
if len(categories) > 0 {
|
||||
fmt.Fprintf(b, "## 内容分类\n\n")
|
||||
for _, c := range categories {
|
||||
if c == nil || c.Id == 0 {
|
||||
continue
|
||||
}
|
||||
name := c.Name
|
||||
if c.Title != "" {
|
||||
name = c.Title
|
||||
}
|
||||
catDesc := c.SeoInfo.Description
|
||||
if catDesc == "" {
|
||||
catDesc = c.Desc
|
||||
}
|
||||
if catDesc == "" {
|
||||
catDesc = name + "相关视频内容"
|
||||
}
|
||||
fmt.Fprintf(b, "- [%s](%s/video/more/%d): %s\n", name, baseURL, c.Id, catDesc)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// 全部标签
|
||||
if len(tags) > 0 {
|
||||
fmt.Fprintf(b, "## 全部标签\n\n")
|
||||
for _, t := range tags {
|
||||
if t == nil || t.Id == 0 {
|
||||
continue
|
||||
}
|
||||
tagDesc := t.SeoInfo.Description
|
||||
if tagDesc == "" {
|
||||
tagDesc = t.Title
|
||||
}
|
||||
fmt.Fprintf(b, "- [%s](%s/video/all?pageNum=1&pageSize=40&tagsIds=%d): %s\n", t.Title, baseURL, t.Id, tagDesc)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// 作者
|
||||
if len(actresses) > 0 {
|
||||
fmt.Fprintf(b, "## 作者\n\n")
|
||||
for _, item := range actresses {
|
||||
if item == nil || item.Id == 0 {
|
||||
continue
|
||||
}
|
||||
itemDesc := item.SeoInfo.Description
|
||||
if itemDesc == "" {
|
||||
itemDesc = item.Desc
|
||||
}
|
||||
fmt.Fprintf(b, "- [%s](%s/actress/detail?id=%d): %s\n",
|
||||
item.Alias, baseURL, item.Id, itemDesc)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
// 视频详情列表
|
||||
if len(videos) > 0 {
|
||||
fmt.Fprintf(b, "## 内容详情(最新 %d 条)\n\n", len(videos))
|
||||
for _, v := range videos {
|
||||
if v == nil || v.Id == 0 {
|
||||
continue
|
||||
}
|
||||
url := utils.JoinUrlPath(baseURL, fmt.Sprintf("/video/detail?id=%d", v.Id))
|
||||
fmt.Fprintf(b, "### [%s](%s)\n\n", v.Title, url)
|
||||
|
||||
if v.Desc != "" {
|
||||
fmt.Fprintf(b, "%s\n\n", truncate(v.Desc, 500))
|
||||
}
|
||||
|
||||
if !v.AddedTime.IsZero() {
|
||||
fmt.Fprintf(b, "- 发布时间:%s\n", v.AddedTime.Format("2006-01-02"))
|
||||
}
|
||||
if v.CreatorName != "" {
|
||||
fmt.Fprintf(b, "- 创作者:%s\n", v.CreatorName)
|
||||
}
|
||||
if len(v.Keyworld) > 0 {
|
||||
fmt.Fprintf(b, "- 关键词:%s\n", strings.Join(v.Keyworld, "、"))
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeImportantPages(b *strings.Builder, baseURL string) {
|
||||
fmt.Fprintf(b, "## 重要页面\n\n")
|
||||
fmt.Fprintf(b, "- [首页](%s/): 网站主页\n", baseURL)
|
||||
fmt.Fprintf(b, "- [热门搜索](%s/search/hot): 热门搜索关键词\n", baseURL)
|
||||
fmt.Fprintf(b, "- [视频首页](%s/video): 高清视频内容入口\n", baseURL)
|
||||
fmt.Fprintf(b, "- [全部视频](%s/video/all): 浏览全部视频\n", baseURL)
|
||||
fmt.Fprintf(b, "- [视频排行榜](%s/video/rank): 热门视频排行\n", baseURL)
|
||||
fmt.Fprintf(b, "- [线路导航](%s/nav): 最新可用线路入口\n", baseURL)
|
||||
fmt.Fprintf(b, "- [应用导航](%s/application): 应用推荐与下载入口\n", baseURL)
|
||||
fmt.Fprintf(b, "- [RSS订阅](%s/rss.xml): RSS Feed\n", baseURL)
|
||||
fmt.Fprintf(b, "- [Sitemap](%s/sitemap.xml): 网站地图\n\n", baseURL)
|
||||
}
|
||||
|
||||
// ── 工具函数 ──────────────────────────────────────────────────
|
||||
|
||||
func truncate(s string, maxLen int) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return string(runes[:maxLen]) + "…"
|
||||
}
|
||||
|
||||
func writeTextFile(outDir, filename, content string) error {
|
||||
if err := os.MkdirAll(outDir, 0755); err != nil {
|
||||
return fmt.Errorf("mkdir %s: %w", outDir, err)
|
||||
}
|
||||
tmpPath := filepath.Join(outDir, filename+".tmp")
|
||||
f, err := os.Create(tmpPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create %s: %w", tmpPath, err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := f.WriteString(content); err != nil {
|
||||
return fmt.Errorf("write %s: %w", filename, err)
|
||||
}
|
||||
return f.Sync()
|
||||
}
|
||||
|
||||
func commitFile(outDir, filename string) error {
|
||||
src := filepath.Join(outDir, filename+".tmp")
|
||||
dst := filepath.Join(outDir, filename)
|
||||
if err := os.Rename(src, dst); err != nil {
|
||||
os.Remove(src)
|
||||
return fmt.Errorf("rename %s: %w", filename, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package llmstxt
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWriteImportantPagesUsesWebRoutes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var b strings.Builder
|
||||
writeImportantPages(&b, "https://91jxporn.com")
|
||||
out := b.String()
|
||||
|
||||
for _, forbidden := range []string{"/tag/all", "/comicsList", "/search/hot/"} {
|
||||
if strings.Contains(out, forbidden) {
|
||||
t.Fatalf("important pages should not contain legacy path %q:\n%s", forbidden, out)
|
||||
}
|
||||
}
|
||||
|
||||
for _, required := range []string{
|
||||
"/search/hot)",
|
||||
"/video)",
|
||||
"/nav)",
|
||||
"/application)",
|
||||
} {
|
||||
if !strings.Contains(out, required) {
|
||||
t.Fatalf("important pages missing %q:\n%s", required, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user