rootandClaude Opus 5.5
7974a42301
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
387 lines
12 KiB
Go
387 lines
12 KiB
Go
package llmstxt
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"videoweb/pub/common/log"
|
|
"videoweb/pub/common/utils"
|
|
"videoweb/pub/pubdef"
|
|
"videoweb/web/service/srv_cfg"
|
|
)
|
|
|
|
const (
|
|
maxRecentToolsInBrief = 50 // llms.txt 中最新工具条数上限
|
|
maxRecentArticlesBrief = 50 // llms.txt 中最新文章条数上限
|
|
maxRecentInFull = 500 // llms-full.txt 中最新工具/文章条数上限
|
|
)
|
|
|
|
var generateMu sync.Mutex
|
|
|
|
// GenerateLLMSTxt 生成 llms.txt（摘要）和 llms-full.txt（详细）
|
|
func GenerateLLMSTxt() {
|
|
generateMu.Lock()
|
|
defer generateMu.Unlock()
|
|
|
|
ctx := context.Background()
|
|
config, serr := srv_cfg.GetSystemConfig(ctx)
|
|
if serr != nil || config == nil {
|
|
log.Error("GenerateLLMSTxt GetSystemConfig", log.Any("err", serr))
|
|
return
|
|
}
|
|
|
|
//baseURL := strings.TrimSuffix(config.WebAddr, "/")
|
|
|
|
//categories, subCategories, cateNameMap, subCateNameMap := loadCategories(ctx)
|
|
//recentTools := loadRecentTools(ctx, maxRecentToolsInBrief)
|
|
//recentArticles := loadRecentArticles(ctx, maxRecentArticlesBrief)
|
|
//
|
|
//var brief strings.Builder
|
|
//writeBrief(&brief, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld, recentTools, recentArticles)
|
|
//
|
|
//if err := writeTextFile("llms.txt", brief.String()); err != nil {
|
|
// log.Error("GenerateLLMSTxt write llms.txt", log.Any("err", err))
|
|
// return
|
|
//}
|
|
//if err := commitFile("llms.txt"); err != nil {
|
|
// log.Error("GenerateLLMSTxt commit llms.txt", log.Any("err", err))
|
|
// return
|
|
//}
|
|
//
|
|
//allTools := loadRecentTools(ctx, maxRecentInFull)
|
|
//allArticles := loadRecentArticles(ctx, maxRecentInFull)
|
|
//
|
|
//var full strings.Builder
|
|
//writeFull(&full, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld, categories, subCategories, cateNameMap, subCateNameMap, allTools, allArticles)
|
|
//
|
|
//if err := writeTextFile("llms-full.txt", full.String()); err != nil {
|
|
// log.Error("GenerateLLMSTxt write llms-full.txt", log.Any("err", err))
|
|
// return
|
|
//}
|
|
//if err := commitFile("llms-full.txt"); err != nil {
|
|
// log.Error("GenerateLLMSTxt commit llms-full.txt", log.Any("err", err))
|
|
// return
|
|
//}
|
|
|
|
//log.Info("GenerateLLMSTxt done",
|
|
// log.Any("categories", len(categories)),
|
|
// log.Any("subCategories", len(subCategories)),
|
|
// log.Any("tools", len(allTools)),
|
|
// log.Any("articles", len(allArticles)),
|
|
//)
|
|
}
|
|
|
|
// ── 数据加载 ──────────────────────────────────────────────────
|
|
//
|
|
//func loadCategories(ctx context.Context) (
|
|
// categories toolcategorymod.ToolCategoryModelList,
|
|
// subCategories toolsubcategorymod.ToolSubCategoryModelList,
|
|
// cateNameMap map[int64]string,
|
|
// subCateNameMap map[int64]string,
|
|
//) {
|
|
// cateFilter := bson.M{
|
|
// toolcategorymod.FN.ToolCategory.Status: 1,
|
|
// toolcategorymod.FN.ToolCategory.Deleted: bson.M{"$ne": 1},
|
|
// }
|
|
// cateOpt := options.Find().SetSort(bson.D{
|
|
// {Key: toolcategorymod.FN.ToolCategory.Sort, Value: 1},
|
|
// {Key: toolcategorymod.FN.ToolCategory.Id, Value: 1},
|
|
// })
|
|
// categories, _ = toolcategorymod.FindMany(ctx, cateFilter, cateOpt)
|
|
//
|
|
// subFilter := bson.M{
|
|
// toolsubcategorymod.FN.ToolSubCategory.Status: 1,
|
|
// toolsubcategorymod.FN.ToolSubCategory.Deleted: bson.M{"$ne": 1},
|
|
// }
|
|
// subOpt := options.Find().SetSort(bson.D{
|
|
// {Key: toolsubcategorymod.FN.ToolSubCategory.Sort, Value: 1},
|
|
// {Key: toolsubcategorymod.FN.ToolSubCategory.Id, Value: 1},
|
|
// })
|
|
// subCategories, _ = toolsubcategorymod.FindMany(ctx, subFilter, subOpt)
|
|
//
|
|
// cateNameMap = make(map[int64]string, len(categories))
|
|
// for _, c := range categories {
|
|
// cateNameMap[c.Id] = c.Name
|
|
// }
|
|
// subCateNameMap = make(map[int64]string, len(subCategories))
|
|
// for _, s := range subCategories {
|
|
// subCateNameMap[s.Id] = s.Name
|
|
// }
|
|
// return categories, subCategories, cateNameMap, subCateNameMap
|
|
//}
|
|
//
|
|
//func loadRecentTools(ctx context.Context, limit int64) toolmod.ToolModelList {
|
|
// opt := options.Find().
|
|
// SetSort(bson.D{
|
|
// {Key: toolmod.FN.Tool.Sort, Value: 1},
|
|
// {Key: toolmod.FN.Tool.Hot, Value: -1},
|
|
// {Key: toolmod.FN.Tool.CreatedAt, Value: -1},
|
|
// }).
|
|
// SetLimit(limit)
|
|
// filter := bson.M{
|
|
// toolmod.FN.Tool.ReviewStatus: 1,
|
|
// toolmod.FN.Tool.Slug: bson.M{"$nin": []interface{}{"", nil}},
|
|
// }
|
|
// list, _ := toolmod.FindMany(ctx, filter, opt)
|
|
// return list
|
|
//}
|
|
//
|
|
//func loadRecentArticles(ctx context.Context, limit int64) articlemod.ArticleModelList {
|
|
// opt := options.Find().
|
|
// SetSort(bson.D{{Key: articlemod.FN.Article.CreatedAt, Value: -1}}).
|
|
// SetLimit(limit)
|
|
// filter := bson.M{
|
|
// articlemod.FN.Article.ReviewStatus: 1,
|
|
// articlemod.FN.Article.Slug: bson.M{"$nin": []interface{}{"", nil}},
|
|
// }
|
|
// list, _ := articlemod.FindMany(ctx, filter, opt)
|
|
// return list
|
|
//}
|
|
//
|
|
//// ── 内容构建 ──────────────────────────────────────────────────
|
|
//
|
|
//func writeBrief(
|
|
// b *strings.Builder,
|
|
// title, desc, baseURL string,
|
|
// keywords []string,
|
|
// recentTools toolmod.ToolModelList,
|
|
// recentArticles articlemod.ArticleModelList,
|
|
//) {
|
|
// writeHeader(b, title, desc, keywords)
|
|
// writeImportantPages(b, baseURL)
|
|
//
|
|
// if len(recentTools) > 0 {
|
|
// fmt.Fprintf(b, "## 最新 AI 工具\n\n")
|
|
// for _, t := range recentTools {
|
|
// if t == nil || t.Slug == "" {
|
|
// continue
|
|
// }
|
|
// summary := t.Intro
|
|
// if summary == "" {
|
|
// summary = t.Desc
|
|
// }
|
|
// if summary == "" {
|
|
// summary = t.Title
|
|
// }
|
|
// fmt.Fprintf(b, "- [%s](%s): %s\n", t.Title, toolDetailURL(baseURL, t.Slug), truncate(summary, 100))
|
|
// }
|
|
// b.WriteString("\n")
|
|
// }
|
|
//
|
|
// if len(recentArticles) > 0 {
|
|
// fmt.Fprintf(b, "## 最新 AI 资讯\n\n")
|
|
// for _, a := range recentArticles {
|
|
// if a == nil || a.Slug == "" {
|
|
// continue
|
|
// }
|
|
// summary := a.Intro
|
|
// if summary == "" {
|
|
// summary = a.Title
|
|
// }
|
|
// fmt.Fprintf(b, "- [%s](%s): %s\n", a.Title, articleDetailURL(baseURL, a.Slug), truncate(summary, 100))
|
|
// }
|
|
// b.WriteString("\n")
|
|
// }
|
|
//}
|
|
//
|
|
//func GetCategorySlug(id int64, categories toolcategorymod.ToolCategoryModelList) string {
|
|
// if id <= 0 {
|
|
// return ""
|
|
// }
|
|
// if len(categories) == 0 {
|
|
// return ""
|
|
// }
|
|
// for _, category := range categories {
|
|
// if category.Id == id {
|
|
// return category.Slug
|
|
// }
|
|
// }
|
|
// return ""
|
|
//}
|
|
//func GetSubCategorySlug(id int64, categories toolsubcategorymod.ToolSubCategoryModelList) string {
|
|
// if id <= 0 {
|
|
// return ""
|
|
// }
|
|
// if len(categories) == 0 {
|
|
// return ""
|
|
// }
|
|
// for _, category := range categories {
|
|
// if category.Id == id {
|
|
// return category.Slug
|
|
// }
|
|
// }
|
|
// return ""
|
|
//}
|
|
//func writeFull(
|
|
// b *strings.Builder,
|
|
// title, desc, baseURL string,
|
|
// keywords []string,
|
|
// categories toolcategorymod.ToolCategoryModelList,
|
|
// subCategories toolsubcategorymod.ToolSubCategoryModelList,
|
|
// cateNameMap, subCateNameMap map[int64]string,
|
|
// tools toolmod.ToolModelList,
|
|
// articles articlemod.ArticleModelList,
|
|
//) {
|
|
// writeHeader(b, title, desc, keywords)
|
|
// writeImportantPages(b, baseURL)
|
|
// writeCategorySections(b, baseURL, categories, subCategories, true)
|
|
//
|
|
// if len(tools) > 0 {
|
|
// fmt.Fprintf(b, "## AI 工具详情（最新 %d 条）\n\n", len(tools))
|
|
// for _, t := range tools {
|
|
// if t == nil || t.Slug == "" {
|
|
// continue
|
|
// }
|
|
// fmt.Fprintf(b, "### [%s](%s)\n\n", t.Title, toolDetailURL(baseURL, t.Slug))
|
|
//
|
|
// summary := t.Desc
|
|
// if summary == "" {
|
|
// summary = t.Intro
|
|
// }
|
|
// if summary != "" {
|
|
// fmt.Fprintf(b, "%s\n\n", truncate(summary, 500))
|
|
// }
|
|
// if subName := subCateNameMap[t.SubCateId]; subName != "" {
|
|
// fmt.Fprintf(b, "- 子分类：[%s](%s)\n", subName, subCategoryListURL(baseURL, GetCategorySlug(t.CategoryId, categories), GetSubCategorySlug(t.SubCateId, subCategories)))
|
|
// }
|
|
// if t.OfficialUrl != "" {
|
|
// fmt.Fprintf(b, "- 官网：%s\n", t.OfficialUrl)
|
|
// }
|
|
// if len(t.SeoInfo.Keyworld) > 0 {
|
|
// fmt.Fprintf(b, "- 关键词：%s\n", strings.Join(t.SeoInfo.Keyworld, "、"))
|
|
// }
|
|
// if !t.CreatedAt.IsZero() {
|
|
// fmt.Fprintf(b, "- 发布时间：%s\n", t.CreatedAt.Format("2006-01-02"))
|
|
// }
|
|
// b.WriteString("\n")
|
|
// }
|
|
// }
|
|
//
|
|
// if len(articles) > 0 {
|
|
// fmt.Fprintf(b, "## AI 资讯详情（最新 %d 条）\n\n", len(articles))
|
|
// for _, a := range articles {
|
|
// if a == nil || a.Slug == "" {
|
|
// continue
|
|
// }
|
|
// fmt.Fprintf(b, "### [%s](%s)\n\n", a.Title, articleDetailURL(baseURL, a.Slug))
|
|
//
|
|
// if a.Intro != "" {
|
|
// fmt.Fprintf(b, "%s\n\n", truncate(a.Intro, 500))
|
|
// }
|
|
// if a.Author != "" {
|
|
// fmt.Fprintf(b, "- 作者：%s\n", a.Author)
|
|
// }
|
|
// if len(a.SeoInfo.Keyworld) > 0 {
|
|
// fmt.Fprintf(b, "- 关键词：%s\n", strings.Join(a.SeoInfo.Keyworld, "、"))
|
|
// }
|
|
// if !a.CreatedAt.IsZero() {
|
|
// fmt.Fprintf(b, "- 发布时间：%s\n", a.CreatedAt.Format("2006-01-02"))
|
|
// }
|
|
// b.WriteString("\n")
|
|
// }
|
|
// }
|
|
//}
|
|
//
|
|
//func writeHeader(b *strings.Builder, title, desc string, keywords []string) {
|
|
// now := time.Now().Format("2006-01-02")
|
|
// fmt.Fprintf(b, "# %s\n\n", title)
|
|
// if desc != "" {
|
|
// fmt.Fprintf(b, "> %s\n\n", desc)
|
|
// }
|
|
// if len(keywords) > 0 {
|
|
// fmt.Fprintf(b, "> 关键词：%s\n\n", strings.Join(keywords, "、"))
|
|
// }
|
|
// fmt.Fprintf(b, "> 更新时间：%s\n\n", now)
|
|
//}
|
|
//
|
|
//func writeImportantPages(b *strings.Builder, baseURL string) {
|
|
// fmt.Fprintf(b, "## 重要页面\n\n")
|
|
// fmt.Fprintf(b, "- [首页](%s/): AI 工具导航主页\n", baseURL)
|
|
// fmt.Fprintf(b, "- [工具列表](%s/tool/list): 全部 AI 工具\n", baseURL)
|
|
// fmt.Fprintf(b, "- [热门推荐](%s/tool/hot): 热门 AI 工具\n", baseURL)
|
|
// fmt.Fprintf(b, "- [AI 资讯](%s/article/list): AI 行业资讯文章\n", baseURL)
|
|
// fmt.Fprintf(b, "- [搜索](%s/search): 搜索工具与资讯\n", baseURL)
|
|
// fmt.Fprintf(b, "- [关于我们](%s/about/): 站点介绍\n", baseURL)
|
|
// fmt.Fprintf(b, "- [Sitemap](%s/sitemap.xml): 网站地图\n\n", baseURL)
|
|
//}
|
|
//
|
|
//func writeCategorySections(
|
|
// b *strings.Builder,
|
|
// baseURL string,
|
|
// categories toolcategorymod.ToolCategoryModelList,
|
|
// subCategories toolsubcategorymod.ToolSubCategoryModelList,
|
|
// withSeo bool,
|
|
//) {
|
|
//
|
|
// if len(subCategories) > 0 {
|
|
// fmt.Fprintf(b, "## 工具子分类\n\n")
|
|
// for _, s := range subCategories {
|
|
// if s == nil {
|
|
// continue
|
|
// }
|
|
// subDesc := subCategoryDesc(s, withSeo)
|
|
// fmt.Fprintf(b, "- [%s](%s): %s\n", s.Name, subCategoryListURL(baseURL, GetCategorySlug(s.CategoryId, categories), GetSubCategorySlug(s.Id, subCategories)), subDesc)
|
|
// }
|
|
// b.WriteString("\n")
|
|
// }
|
|
//}
|
|
//
|
|
//func subCategoryDesc(s *toolsubcategorymod.ToolSubCategoryModel, withSeo bool) string {
|
|
// if withSeo && s.SeoInfo.Description != "" {
|
|
// return s.SeoInfo.Description
|
|
// }
|
|
// return s.Name + "相关 AI 工具"
|
|
//}
|
|
|
|
// ── URL 构建 ──────────────────────────────────────────────────
|
|
|
|
func toolDetailURL(baseURL, slug string) string {
|
|
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/tool/details/%s", slug))
|
|
}
|
|
|
|
func articleDetailURL(baseURL, slug string) string {
|
|
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/article/details/%s", slug))
|
|
}
|
|
|
|
func subCategoryListURL(baseURL string, categorySlug, subCateSlug string) string {
|
|
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/classification/%s/%s", categorySlug, subCateSlug))
|
|
}
|
|
|
|
// ── 工具函数 ──────────────────────────────────────────────────
|
|
|
|
func truncate(s string, maxLen int) string {
|
|
runes := []rune(s)
|
|
if len(runes) <= maxLen {
|
|
return s
|
|
}
|
|
return string(runes[:maxLen]) + "…"
|
|
}
|
|
|
|
func writeTextFile(filename, content string) error {
|
|
if err := os.MkdirAll(pubdef.OutputDir, 0755); err != nil {
|
|
return fmt.Errorf("mkdir %s: %w", pubdef.OutputDir, err)
|
|
}
|
|
tmpPath := filepath.Join(pubdef.OutputDir, filename+".tmp")
|
|
f, err := os.Create(tmpPath)
|
|
if err != nil {
|
|
return fmt.Errorf("create %s: %w", tmpPath, err)
|
|
}
|
|
defer f.Close()
|
|
if _, err := f.WriteString(content); err != nil {
|
|
return fmt.Errorf("write %s: %w", filename, err)
|
|
}
|
|
return f.Sync()
|
|
}
|
|
|
|
func commitFile(filename string) error {
|
|
src := filepath.Join(pubdef.OutputDir, filename+".tmp")
|
|
dst := filepath.Join(pubdef.OutputDir, filename)
|
|
if err := os.Rename(src, dst); err != nil {
|
|
os.Remove(src)
|
|
return fmt.Errorf("rename %s: %w", filename, err)
|
|
}
|
|
return nil
|
|
}
|