Initial commit: web_hghl_server

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-10-26 16:28:14 +08:00
co-authored by Claude Opus 4.7
commit 65fca60622
653 changed files with 135901 additions and 0 deletions
+58
View File
@@ -0,0 +1,58 @@
package ipstat
import (
"context"
"time"
"videoweb/pub/common/enum/redisconst"
"videoweb/pub/common/log"
"videoweb/pub/common/utils"
"videoweb/pub/models/ipdailystatmod"
"videoweb/pub/pubdef"
"videoweb/pub/redis"
)
const syncInterval = 5 * time.Minute
// StartSync 启动后台协程,每隔 syncInterval 将当日和昨日的 Redis IP SET
// 数量同步写入 MongoDB(ip_daily_stat),供 web 端查询。
func StartSync() {
go func() {
// 启动时立即同步一次
syncOnce()
ticker := time.NewTicker(syncInterval)
defer ticker.Stop()
for range ticker.C {
syncOnce()
}
}()
}
func syncOnce() {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
now := time.Now().In(utils.LocBJ)
appId := int64(pubdef.AppId)
for _, offset := range []int{0, -1} {
date := now.AddDate(0, 0, offset).Format("20060102")
key := redisconst.IpStatKey(appId, date)
count, err := redis.SCard(key)
if err != nil {
log.Warn("ipstat sync: SCard failed",
log.Any("date", date), log.E(err))
continue
}
// 今日 Redis 为 0 时不覆盖 MongoDB,避免 key 丢失把已有数据写成 0。
if offset == 0 && count == 0 {
continue
}
if err := ipdailystatmod.UpsertCount(ctx, appId, date, count); err != nil {
log.Warn("ipstat sync: UpsertCount failed",
log.Any("appId", appId), log.Any("date", date), log.E(err))
}
}
}
+134
View File
@@ -0,0 +1,134 @@
package dhsitemap
import (
"context"
"encoding/xml"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"videoweb/app/service/configser"
"videoweb/pub/common/log"
"videoweb/pub/common/utils"
)
const (
outputDir = "../h5/templates/page/sitemaps"
filename = "dhsitemap.xml"
// 与主站 sitemap 一致:浏览器用 XSLT 把 loc 渲成可点击链接
stylesheetURLSet = `<?xml-stylesheet type="text/xsl" href="/static/sitemap.xsl"?>` + "\n"
)
var generateMu sync.Mutex
type urlSet struct {
XMLName xml.Name `xml:"urlset"`
Xmlns string `xml:"xmlns,attr"`
URLs []urlEntry `xml:"url"`
}
type urlEntry struct {
Loc string `xml:"loc"`
LastMod string `xml:"lastmod"`
}
// GenerateDHSitemap 生成导航站 sitemap,域名取自系统配置 newUrlAddr。
func GenerateDHSitemap() {
generateMu.Lock()
defer generateMu.Unlock()
ctx := context.Background()
config, serr := configser.GetSystemConfig(ctx)
if serr != nil {
log.Error("GenerateDHSitemap GetSystemConfig", log.Any("err", serr))
return
}
if config == nil {
log.Error("GenerateDHSitemap GetSystemConfig config is nil")
return
}
urls := collectDHURLs(config.NewUrlAddr, config.NewUrlAddr1, config.NewUrlAddr2)
if len(urls) == 0 {
log.Warn("GenerateDHSitemap: no newUrlAddr configured, skip")
return
}
today := time.Now().Format("2006-01-02")
entries := make([]urlEntry, 0, len(urls))
for _, u := range urls {
entries = append(entries, urlEntry{
Loc: u,
LastMod: today,
})
}
doc := urlSet{
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
URLs: entries,
}
if err := writeFile(doc); err != nil {
log.Error("GenerateDHSitemap write", log.Any("err", err))
return
}
src := filepath.Join(outputDir, filename+".tmp")
dst := filepath.Join(outputDir, filename)
if err := os.Rename(src, dst); err != nil {
log.Error("GenerateDHSitemap rename", log.Any("err", err))
os.Remove(src)
return
}
log.Info("GenerateDHSitemap done", log.Any("urls", len(entries)))
}
func collectDHURLs(addrs ...string) []string {
seen := make(map[string]struct{}, len(addrs))
urls := make([]string, 0, len(addrs))
for _, addr := range addrs {
addr = strings.TrimSpace(addr)
if addr == "" {
continue
}
loc := utils.JoinUrlPath(addr, "/")
if _, ok := seen[loc]; ok {
continue
}
seen[loc] = struct{}{}
urls = append(urls, loc)
}
return urls
}
func writeFile(doc urlSet) error {
if err := os.MkdirAll(outputDir, 0755); err != nil {
return fmt.Errorf("mkdir %s: %w", outputDir, err)
}
path := filepath.Join(outputDir, filename+".tmp")
file, err := os.Create(path)
if err != nil {
return fmt.Errorf("create %s: %w", filename, err)
}
defer file.Close()
if _, err := file.WriteString(xml.Header); err != nil {
return fmt.Errorf("write header: %w", err)
}
if _, err := file.WriteString(stylesheetURLSet); err != nil {
return fmt.Errorf("write stylesheet: %w", err)
}
encoder := xml.NewEncoder(file)
encoder.Indent("", " ")
if err := encoder.Encode(doc); err != nil {
return fmt.Errorf("encode: %w", err)
}
if err := encoder.Flush(); err != nil {
return fmt.Errorf("flush: %w", err)
}
return file.Sync()
}
+362
View File
@@ -0,0 +1,362 @@
package llmstxt
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"videoweb/app/service/configser"
"videoweb/pub/common/log"
"videoweb/pub/common/utils"
"videoweb/pub/models/actress"
"videoweb/pub/models/videocategorymod"
"videoweb/pub/models/videoinformationmod"
"videoweb/pub/models/videotagmod"
"videoweb/pub/pubdef"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
maxTagsInBrief = 80 // llms.txt 中标签条数上限
maxRecentInFull = 500 // llms-full.txt 中最新视频条数上限
)
var generateMu sync.Mutex
// GenerateLLMSTxt 生成 llms.txt(摘要)和 llms-full.txt(详细),输出到 outDir。
func GenerateLLMSTxt(outDir string) {
generateMu.Lock()
defer generateMu.Unlock()
ctx := context.Background()
config, serr := configser.GetSystemConfig(ctx)
if serr != nil || config == nil {
log.Error("GenerateLLMSTxt GetSystemConfig", log.Any("err", serr))
return
}
baseURL := strings.TrimSuffix(config.WebAddr, "/")
// ── 查询数据 ──────────────────────────────────────────────
categoryList, _ := videocategorymod.FindManyVideoCategorys(ctx,
bson.M{videocategorymod.FN.VideoCategory.Status: true},
func() *options.FindOptions {
o := options.Find()
o.SetSort(bson.M{videocategorymod.FN.VideoCategory.Rank: -1})
return o
}())
tagOpt := options.Find()
tagOpt.SetSort(bson.M{videotagmod.FN.VideoTagInfo.WatchTimes: -1})
tagOpt.SetLimit(int64(maxTagsInBrief))
tagList, _ := videotagmod.FindManyVideoTagInfos(ctx,
bson.M{videotagmod.FN.VideoTagInfo.Status: true}, tagOpt)
actressOpt := options.Find()
actressOpt.SetSort(bson.M{actress.FN.Actress.Works: -1})
actressList, _ := actress.FindManyActresss(ctx,
bson.M{actress.FN.Actress.Status: true}, actressOpt)
recentOpt := options.Find()
recentOpt.SetSort(bson.M{videoinformationmod.FN.VideoInformation.AddedTime: -1})
recentOpt.SetLimit(50)
recentVideos, _ := videoinformationmod.FindManyVideoInformations(ctx,
bson.M{videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK},
recentOpt)
// ── 生成 llms.txt ─────────────────────────────────────────
var brief strings.Builder
writeBrief(&brief, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld,
categoryList, tagList, actressList, recentVideos)
if err := writeTextFile(outDir, "llms.txt", brief.String()); err != nil {
log.Error("GenerateLLMSTxt write llms.txt", log.Any("err", err))
return
}
if err := commitFile(outDir, "llms.txt"); err != nil {
log.Error("GenerateLLMSTxt commit llms.txt", log.Any("err", err))
return
}
// ── 生成 llms-full.txt ────────────────────────────────────
allVideoOpt := options.Find()
allVideoOpt.SetSort(bson.M{videoinformationmod.FN.VideoInformation.AddedTime: -1})
allVideoOpt.SetLimit(int64(maxRecentInFull))
allVideos, _ := videoinformationmod.FindManyVideoInformations(ctx,
bson.M{videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK},
allVideoOpt)
var full strings.Builder
writeFull(&full, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld,
categoryList, tagList, actressList, allVideos)
if err := writeTextFile(outDir, "llms-full.txt", full.String()); err != nil {
log.Error("GenerateLLMSTxt write llms-full.txt", log.Any("err", err))
return
}
if err := commitFile(outDir, "llms-full.txt"); err != nil {
log.Error("GenerateLLMSTxt commit llms-full.txt", log.Any("err", err))
return
}
log.Info("GenerateLLMSTxt done",
log.Any("categories", len(categoryList)),
log.Any("tags", len(tagList)),
log.Any("actresses", len(actressList)),
log.Any("videos", len(allVideos)),
)
}
// ── 内容构建 ──────────────────────────────────────────────────
func writeBrief(
b *strings.Builder,
title, desc, baseURL string,
keywords []string,
categories videocategorymod.VideoCategoryList,
tags videotagmod.VideoTagInfoList,
actresses actress.ActressList,
recentVideos videoinformationmod.VideoInformationList,
) {
now := time.Now().Format("2006-01-02")
// 文件头
fmt.Fprintf(b, "# %s\n\n", title)
if desc != "" {
fmt.Fprintf(b, "> %s\n\n", desc)
}
if len(keywords) > 0 {
fmt.Fprintf(b, "> 关键词:%s\n\n", strings.Join(keywords, "、"))
}
fmt.Fprintf(b, "> 更新时间:%s\n\n", now)
// 重要页面
writeImportantPages(b, baseURL)
// 分类
if len(categories) > 0 {
fmt.Fprintf(b, "## 内容分类\n\n")
for _, c := range categories {
if c == nil || c.Id == 0 {
continue
}
name := c.Name
if c.Title != "" {
name = c.Title
}
catDesc := c.Desc
if catDesc == "" {
catDesc = name + "相关视频内容"
}
fmt.Fprintf(b, "- [%s](%s/video/more/%d): %s\n", name,
baseURL, c.Id, catDesc)
}
b.WriteString("\n")
}
// 热门标签(取播放量最高的)
if len(tags) > 0 {
fmt.Fprintf(b, "## 热门标签\n\n")
for _, t := range tags {
if t == nil || t.Id == 0 {
continue
}
fmt.Fprintf(b, "- [%s](%s/video/all?pageNum=1&pageSize=40&tagsIds=%d)\n", t.Title, baseURL, t.Id)
}
b.WriteString("\n")
}
// 作者
if len(actresses) > 0 {
fmt.Fprintf(b, "## 作者\n\n")
for _, item := range actresses {
if item == nil || item.Id == 0 {
continue
}
fmt.Fprintf(b, "- [%s](%s/actress/detail?id=%d)\n", item.Alias, baseURL, item.Id)
}
b.WriteString("\n")
}
// 最新内容(50条)
if len(recentVideos) > 0 {
fmt.Fprintf(b, "## 最新内容\n\n")
for _, v := range recentVideos {
if v == nil || v.Id == 0 {
continue
}
url := utils.JoinUrlPath(baseURL, fmt.Sprintf("/video/detail?id=%d", v.Id))
desc := v.Desc
if desc == "" {
desc = v.Title
}
desc = truncate(desc, 100)
fmt.Fprintf(b, "- [%s](%s): %s\n", v.Title, url, desc)
}
b.WriteString("\n")
}
}
func writeFull(
b *strings.Builder,
title, desc, baseURL string,
keywords []string,
categories videocategorymod.VideoCategoryList,
tags videotagmod.VideoTagInfoList,
actresses actress.ActressList,
videos videoinformationmod.VideoInformationList,
) {
// llms-full.txt 先包含 brief 的所有内容(用全量视频列表替换最新50条)
// 再附加每条视频的完整详情
now := time.Now().Format("2006-01-02")
fmt.Fprintf(b, "# %s\n\n", title)
if desc != "" {
fmt.Fprintf(b, "> %s\n\n", desc)
}
if len(keywords) > 0 {
fmt.Fprintf(b, "> 关键词:%s\n\n", strings.Join(keywords, "、"))
}
fmt.Fprintf(b, "> 更新时间:%s\n\n", now)
// 重要页面
writeImportantPages(b, baseURL)
// 分类(含 SEO 描述)
if len(categories) > 0 {
fmt.Fprintf(b, "## 内容分类\n\n")
for _, c := range categories {
if c == nil || c.Id == 0 {
continue
}
name := c.Name
if c.Title != "" {
name = c.Title
}
catDesc := c.SeoInfo.Description
if catDesc == "" {
catDesc = c.Desc
}
if catDesc == "" {
catDesc = name + "相关视频内容"
}
fmt.Fprintf(b, "- [%s](%s/video/more/%d): %s\n", name, baseURL, c.Id, catDesc)
}
b.WriteString("\n")
}
// 全部标签
if len(tags) > 0 {
fmt.Fprintf(b, "## 全部标签\n\n")
for _, t := range tags {
if t == nil || t.Id == 0 {
continue
}
tagDesc := t.SeoInfo.Description
if tagDesc == "" {
tagDesc = t.Title
}
fmt.Fprintf(b, "- [%s](%s/video/all?pageNum=1&pageSize=40&tagsIds=%d): %s\n", t.Title, baseURL, t.Id, tagDesc)
}
b.WriteString("\n")
}
// 作者
if len(actresses) > 0 {
fmt.Fprintf(b, "## 作者\n\n")
for _, item := range actresses {
if item == nil || item.Id == 0 {
continue
}
itemDesc := item.SeoInfo.Description
if itemDesc == "" {
itemDesc = item.Desc
}
fmt.Fprintf(b, "- [%s](%s/actress/detail?id=%d): %s\n",
item.Alias, baseURL, item.Id, itemDesc)
}
b.WriteString("\n")
}
// 视频详情列表
if len(videos) > 0 {
fmt.Fprintf(b, "## 内容详情(最新 %d 条)\n\n", len(videos))
for _, v := range videos {
if v == nil || v.Id == 0 {
continue
}
url := utils.JoinUrlPath(baseURL, fmt.Sprintf("/video/detail?id=%d", v.Id))
fmt.Fprintf(b, "### [%s](%s)\n\n", v.Title, url)
if v.Desc != "" {
fmt.Fprintf(b, "%s\n\n", truncate(v.Desc, 500))
}
if !v.AddedTime.IsZero() {
fmt.Fprintf(b, "- 发布时间:%s\n", v.AddedTime.Format("2006-01-02"))
}
if v.CreatorName != "" {
fmt.Fprintf(b, "- 创作者:%s\n", v.CreatorName)
}
if len(v.Keyworld) > 0 {
fmt.Fprintf(b, "- 关键词:%s\n", strings.Join(v.Keyworld, "、"))
}
b.WriteString("\n")
}
}
}
func writeImportantPages(b *strings.Builder, baseURL string) {
fmt.Fprintf(b, "## 重要页面\n\n")
fmt.Fprintf(b, "- [首页](%s/): 网站主页\n", baseURL)
fmt.Fprintf(b, "- [热门搜索](%s/search/hot): 热门搜索关键词\n", baseURL)
fmt.Fprintf(b, "- [视频首页](%s/video): 高清视频内容入口\n", baseURL)
fmt.Fprintf(b, "- [全部视频](%s/video/all): 浏览全部视频\n", baseURL)
fmt.Fprintf(b, "- [视频排行榜](%s/video/rank): 热门视频排行\n", baseURL)
fmt.Fprintf(b, "- [线路导航](%s/nav): 最新可用线路入口\n", baseURL)
fmt.Fprintf(b, "- [应用导航](%s/application): 应用推荐与下载入口\n", baseURL)
fmt.Fprintf(b, "- [RSS订阅](%s/rss.xml): RSS Feed\n", baseURL)
fmt.Fprintf(b, "- [Sitemap](%s/sitemap.xml): 网站地图\n\n", baseURL)
}
// ── 工具函数 ──────────────────────────────────────────────────
func truncate(s string, maxLen int) string {
runes := []rune(s)
if len(runes) <= maxLen {
return s
}
return string(runes[:maxLen]) + "…"
}
func writeTextFile(outDir, filename, content string) error {
if err := os.MkdirAll(outDir, 0755); err != nil {
return fmt.Errorf("mkdir %s: %w", outDir, err)
}
tmpPath := filepath.Join(outDir, filename+".tmp")
f, err := os.Create(tmpPath)
if err != nil {
return fmt.Errorf("create %s: %w", tmpPath, err)
}
defer f.Close()
if _, err := f.WriteString(content); err != nil {
return fmt.Errorf("write %s: %w", filename, err)
}
return f.Sync()
}
func commitFile(outDir, filename string) error {
src := filepath.Join(outDir, filename+".tmp")
dst := filepath.Join(outDir, filename)
if err := os.Rename(src, dst); err != nil {
os.Remove(src)
return fmt.Errorf("rename %s: %w", filename, err)
}
return nil
}
+31
View File
@@ -0,0 +1,31 @@
package llmstxt
import (
"strings"
"testing"
)
func TestWriteImportantPagesUsesWebRoutes(t *testing.T) {
t.Parallel()
var b strings.Builder
writeImportantPages(&b, "https://91jxporn.com")
out := b.String()
for _, forbidden := range []string{"/tag/all", "/comicsList", "/search/hot/"} {
if strings.Contains(out, forbidden) {
t.Fatalf("important pages should not contain legacy path %q:\n%s", forbidden, out)
}
}
for _, required := range []string{
"/search/hot)",
"/video)",
"/nav)",
"/application)",
} {
if !strings.Contains(out, required) {
t.Fatalf("important pages missing %q:\n%s", required, out)
}
}
}
+217
View File
@@ -0,0 +1,217 @@
package rss
import (
"context"
"encoding/xml"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"videoweb/app/service/configser"
"videoweb/pub/common/log"
"videoweb/pub/common/netutil"
"videoweb/pub/common/utils"
"videoweb/pub/models/cfg"
"videoweb/pub/models/videoinformationmod"
"videoweb/pub/pubdef"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
maxRSSItems = 200 // RSS 条目上限
)
var generateMu sync.Mutex
// RSS 2.0 结构定义
type rssFeed struct {
XMLName xml.Name `xml:"rss"`
Version string `xml:"version,attr"`
Atom string `xml:"xmlns:atom,attr"`
Channel rssChannel `xml:"channel"`
}
type rssChannel struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
Language string `xml:"language"`
LastBuildDate string `xml:"lastBuildDate"`
AtomLink atomLink `xml:"atom:link"`
Items []rssItem `xml:"item"`
}
type atomLink struct {
Href string `xml:"href,attr"`
Rel string `xml:"rel,attr"`
Type string `xml:"type,attr"`
}
type rssItem struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
PubDate string `xml:"pubDate"`
GUID string `xml:"guid"`
Enclosure *rssEnclosure `xml:"enclosure,omitempty"`
}
type rssEnclosure struct {
URL string `xml:"url,attr"`
Type string `xml:"type,attr"`
Length string `xml:"length,attr"`
}
// GenerateRSS 生成 rss.xml,输出到 outDir/rss.xml(Web 侧通过 /rss.xml 静态暴露)
func GenerateRSS(ctx context.Context, outDir string) {
generateMu.Lock()
defer generateMu.Unlock()
scfg, srvErr := configser.GetSystemConfig(ctx)
if srvErr != nil || scfg == nil {
log.Error("GenerateRSS GetSystemConfig", log.Any("err", srvErr))
return
}
// 查询最新的已上架视频
filter := bson.M{
videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK,
}
opt := options.Find().
SetSort(bson.M{videoinformationmod.FN.VideoInformation.AddedTime: -1}).
SetLimit(int64(maxRSSItems))
videoList, vsErr := videoinformationmod.FindManyVideoInformations(ctx, filter, opt)
if vsErr != nil {
log.Error("GenerateRSS FindManyVideoInformations", log.Any("err", vsErr))
return
}
items := buildRSSItems(scfg, videoList)
feedURL := utils.JoinUrlPath(scfg.WebAddr, "/rss.xml")
feed := rssFeed{
Version: "2.0",
Atom: "http://www.w3.org/2005/Atom",
Channel: rssChannel{
Title: scfg.WebTitle,
Link: scfg.WebAddr,
Description: scfg.WebDescription,
Language: "zh-cn",
LastBuildDate: time.Now().Format(time.RFC1123Z),
AtomLink: atomLink{
Href: feedURL,
Rel: "self",
Type: "application/rss+xml",
},
Items: items,
},
}
const filename = "rss.xml"
if err := writeRSSFile(outDir, filename, feed); err != nil {
log.Error("GenerateRSS write", log.Any("err", err))
return
}
// 写入成功,rename .tmp → 正式文件
src := filepath.Join(outDir, filename+".tmp")
dst := filepath.Join(outDir, filename)
if err := os.Rename(src, dst); err != nil {
log.Error("GenerateRSS rename", log.Any("err", err))
os.Remove(src)
return
}
log.Info("GenerateRSS done", log.Any("items", len(items)))
}
// absModIndex 根据 id 确定性选取切片索引,保证结果非负
func absModIndex(id int64, length int) int {
idx := id % int64(length)
if idx < 0 {
idx = -idx
}
return int(idx)
}
func buildRSSItems(scfg *cfg.SysCfg, videoList videoinformationmod.VideoInformationList) []rssItem {
logoURL := utils.JoinUrlPath(scfg.WebAddr, "/static/images/logo.png")
items := make([]rssItem, 0, len(videoList))
for _, v := range videoList {
desc := v.Desc
if desc == "" {
desc = v.Title
}
pageURL := utils.JoinUrlPathKeepSlash(scfg.WebAddr, fmt.Sprintf("/video/detail?id=%d", v.Id))
pubDate := ""
if !v.AddedTime.IsZero() {
pubDate = v.AddedTime.Format(time.RFC1123Z)
}
item := rssItem{
Title: v.Title,
Link: pageURL,
Description: desc,
PubDate: pubDate,
GUID: pageURL,
}
// 封面图优先级:UnencryptedImage > SitemapDefaultImages 确定性选取 > logo
encURL := ""
encType := "image/jpeg"
if v.UnencryptedImage != "" && scfg.UnencryptedCdn != "" {
encURL = netutil.MergeURL(scfg.UnencryptedCdn, v.UnencryptedImage)
} else if len(scfg.SitemapDefaultImages) > 0 && scfg.UnencryptedCdn != "" {
img := scfg.SitemapDefaultImages[absModIndex(v.Id, len(scfg.SitemapDefaultImages))]
encURL = netutil.MergeURL(scfg.UnencryptedCdn, img)
} else {
encURL = logoURL
encType = "image/png"
}
if strings.HasSuffix(encURL, ".png") {
encType = "image/png"
} else if strings.HasSuffix(encURL, ".webp") {
encType = "image/webp"
}
item.Enclosure = &rssEnclosure{
URL: encURL,
Type: encType,
Length: "0",
}
items = append(items, item)
}
return items
}
// writeRSSFile 将数据写入 rssDir/filename.tmp,由调用方负责 rename
func writeRSSFile(rssDir, filename string, data interface{}) error {
if err := os.MkdirAll(rssDir, 0755); err != nil {
return fmt.Errorf("mkdir %s: %w", rssDir, err)
}
path := filepath.Join(rssDir, filename+".tmp")
file, err := os.Create(path)
if err != nil {
return fmt.Errorf("create %s: %w", filename, err)
}
defer file.Close()
if _, err := file.WriteString(xml.Header); err != nil {
return fmt.Errorf("write header %s: %w", filename, err)
}
encoder := xml.NewEncoder(file)
encoder.Indent("", " ")
if err := encoder.Encode(data); err != nil {
return fmt.Errorf("encode %s: %w", filename, err)
}
if err := encoder.Flush(); err != nil {
return fmt.Errorf("flush %s: %w", filename, err)
}
return file.Sync()
}
+312
View File
@@ -0,0 +1,312 @@
package sitemap
import (
"context"
"fmt"
"net/url"
"strconv"
"strings"
"sync"
"time"
"videoweb/pub/common/utils"
"videoweb/pub/models/actress"
"videoweb/pub/models/cfg"
"videoweb/pub/models/translationcategorymod"
"videoweb/pub/models/translationmod"
"videoweb/pub/models/videocategorymod"
"videoweb/pub/models/videoinformationmod"
"videoweb/pub/pubdef"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
// idWithTime 携带 ID、slug 和实际更新时间
type idWithTime struct {
Id int64
Slug string
LastMod time.Time
}
func hasAnyID(ids []int64, used map[int64]struct{}) bool {
for _, id := range ids {
if _, ok := used[id]; ok {
return true
}
}
return false
}
func isSitemapChannelPath(path, prefix string, extras ...string) bool {
if path == prefix || strings.HasPrefix(path, prefix+"/") {
return true
}
for _, extra := range extras {
if path == extra {
return true
}
}
return false
}
func detailSegment(slug string, id int64) string {
if s := strings.TrimSpace(slug); s != "" {
return s
}
if id > 0 {
return strconv.FormatInt(id, 10)
}
return ""
}
func buildChannelKeyURLs(webAddr, channel, keyPrefix string, items []idWithTime, changeFreq string, priority float32) []URL {
urls := make([]URL, 0, len(items))
channel = strings.TrimRight(channel, "/")
for _, item := range items {
seg := detailSegment(item.Slug, item.Id)
if seg == "" {
continue
}
lastMod := item.LastMod.Format(time.DateOnly)
if item.LastMod.IsZero() {
lastMod = time.Now().Format(time.DateOnly)
}
urls = append(urls, URL{
Loc: utils.JoinUrlPathKeepSlash(webAddr, channel+"/"+keyPrefix+seg),
LastMod: lastMod,
ChangeFreq: changeFreq,
Priority: priority,
})
}
return urls
}
// buildCategoryURLs 分类页:/{channel}/c-{slug},无 slug 时回退 c-{id}。
func buildCategoryURLs(webAddr, channel string, items []idWithTime, changeFreq string, priority float32) []URL {
return buildChannelKeyURLs(webAddr, channel, "c-", items, changeFreq, priority)
}
// buildTagURLs 标签页:/{channel}/t-{slug},无 slug 时回退 t-{id}。
func buildTagURLs(webAddr, channel string, items []idWithTime, changeFreq string, priority float32) []URL {
return buildChannelKeyURLs(webAddr, channel, "t-", items, changeFreq, priority)
}
// buildDetailURLs 详情页:/{prefix}/{slug},无 slug 时回退数字 id。
func buildDetailURLs(webAddr, prefix string, items []idWithTime, changeFreq string, priority float32) []URL {
urls := make([]URL, 0, len(items))
for _, item := range items {
seg := detailSegment(item.Slug, item.Id)
if seg == "" {
continue
}
lastMod := item.LastMod.Format(time.DateOnly)
if item.LastMod.IsZero() {
lastMod = time.Now().Format(time.DateOnly)
}
urls = append(urls, URL{
Loc: utils.JoinUrlPathKeepSlash(webAddr, prefix+"/"+seg),
LastMod: lastMod,
ChangeFreq: changeFreq,
Priority: priority,
})
}
return urls
}
// buildURLs 根据 ID+时间列表批量构建 URL 条目
func buildURLs(webAddr string, pathFmt string, items []idWithTime, changeFreq string, priority float32) []URL {
urls := make([]URL, 0, len(items))
for _, item := range items {
lastMod := item.LastMod.Format(time.DateOnly)
if item.LastMod.IsZero() {
lastMod = time.Now().Format(time.DateOnly)
}
urls = append(urls, URL{
Loc: utils.JoinUrlPathKeepSlash(webAddr, fmt.Sprintf(pathFmt, item.Id)),
LastMod: lastMod,
ChangeFreq: changeFreq,
Priority: priority,
})
}
return urls
}
// filterIdsWithTime 返回 source 中存在于 validMap 的条目(含 slug)。
func filterIdsWithTime(source map[int64]struct{}, validMap map[int64]idWithTime) []idWithTime {
result := make([]idWithTime, 0, len(source))
for id := range source {
if item, ok := validMap[id]; ok {
result = append(result, item)
}
}
return result
}
func getVideoCategoryList(ctx context.Context, usedTagIds map[int64]struct{}, hasVideos bool) []idWithTime {
filter := bson.M{
videocategorymod.FN.VideoCategory.Status: true,
}
pageSize := int64(1000)
items := make([]idWithTime, 0)
for i := int64(0); i < 500; i++ {
skip := i * pageSize
list, _ := videocategorymod.FindManyVideoCategorys(ctx, filter, options.Find().SetSort(bson.M{videocategorymod.FN.VideoCategory.Id: -1}).SetSkip(skip).SetLimit(pageSize))
for _, info := range list {
if len(info.Tags) == 0 {
if !hasVideos {
continue
}
} else if !hasAnyID(info.Tags, usedTagIds) {
continue
}
if !videoCategoryHasContent(ctx, info) {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
if len(list) < int(pageSize) {
break
}
}
return items
}
// videoCategoryContentFilter 与 app QueryVideoMore 使用相同的视频筛选条件。
func videoCategoryContentFilter(category *videocategorymod.VideoCategory) bson.M {
filter := bson.M{
videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK,
}
if category != nil && len(category.Tags) > 0 {
filter[videoinformationmod.FN.VideoInformation.Tags] = bson.M{"$in": category.Tags}
}
return filter
}
func videoCategoryHasContent(ctx context.Context, category *videocategorymod.VideoCategory) bool {
if category == nil || !category.Status {
return false
}
count, err := videoinformationmod.CountVideoInformations(ctx, videoCategoryContentFilter(category))
return err == nil && count > 0
}
func getActress(ctx context.Context) map[int64]idWithTime {
filter := bson.M{
actress.FN.Actress.Status: true,
}
pageSize := int64(1000)
pageIds := make(map[int64]idWithTime)
for i := int64(0); i < 100; i++ {
skip := i * pageSize
list, _ := actress.FindManyActresss(ctx, filter, options.Find().SetSkip(skip).SetLimit(pageSize).SetSort(bson.M{actress.FN.Actress.Id: -1}))
for _, info := range list {
pageIds[info.Id] = idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}
}
if len(list) < int(pageSize) {
break
}
}
return pageIds
}
func videoEntriesToIDWithTime(entries []videoEntry) []idWithTime {
items := make([]idWithTime, len(entries))
for i, e := range entries {
items[i] = idWithTime{Id: e.Id, Slug: e.Slug, LastMod: e.LastMod}
}
return items
}
func getVideoList(ctx context.Context) ([]videoEntry, map[int64]struct{}, map[int64]struct{}) {
filter := bson.M{videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK}
pageSize := int64(1000)
items := make([]videoEntry, 0)
tagIds := make(map[int64]struct{})
actressIds := make(map[int64]struct{})
for i := int64(0); i < 500; i++ {
skip := i * pageSize
list, _ := videoinformationmod.FindManyVideoInformations(ctx, filter, options.Find().SetSort(bson.M{videoinformationmod.FN.VideoInformation.Id: -1}).SetLimit(pageSize).SetSkip(skip))
for _, info := range list {
items = append(items, videoEntry{
Id: info.Id,
Slug: info.Slug,
Title: info.Title,
LastMod: info.UpdatedAt,
UnencryptedPreMp4: info.Video.UnencryptedPreMp4,
})
for _, tagId := range info.Tags {
tagIds[tagId] = struct{}{}
}
for _, actressId := range info.ActressIds {
if actressId > 0 {
actressIds[actressId] = struct{}{}
}
}
}
if len(list) < int(pageSize) {
break
}
}
return items, tagIds, actressIds
}
func getTranslation(config cfg.SysCfg, ctx context.Context) []string {
urlList := make([]string, 0)
wg := &sync.WaitGroup{}
wg.Add(2)
var translationCategoryList translationcategorymod.TranslationCategoryList
var translationList translationmod.TranslationInfoList
go func() {
defer wg.Done()
categoryFilter := bson.M{
translationcategorymod.FN.TranslationCategory.Status: true,
}
categoryOpt := &options.FindOptions{}
categoryOpt.SetLimit(1000).SetSort(bson.M{translationcategorymod.FN.TranslationCategory.Rank: -1})
translationCategoryList, _ = translationcategorymod.FindManyTranslationCategorys(ctx, categoryFilter, categoryOpt)
}()
go func() {
defer wg.Done()
translationFilter := bson.M{
translationmod.FN.TranslationInfo.Status: true,
}
translationOpt := &options.FindOptions{}
translationOpt.SetLimit(1000).SetSort(bson.M{translationmod.FN.TranslationInfo.Rank: -1})
translationList, _ = translationmod.FindManyTranslationInfos(ctx, translationFilter, translationOpt)
}()
wg.Wait()
// 按 categoryId 建索引,避免嵌套遍历
translationByCategory := make(map[int64][]*translationmod.TranslationInfo, len(translationList))
for _, t := range translationList {
translationByCategory[t.TransCategoryId] = append(translationByCategory[t.TransCategoryId], t)
}
for _, categoryInfo := range translationCategoryList {
if categoryInfo.CategoryType == pubdef.TranslationTypeUrl {
if strings.HasPrefix(categoryInfo.UrlValue, "http://") || strings.HasPrefix(categoryInfo.UrlValue, "https://") {
continue
}
urlList = append(urlList, utils.JoinUrlPathAppendEnd(config.WebAddr, categoryInfo.UrlValue))
} else if categoryInfo.CategoryType == pubdef.TranslationTypeSub || categoryInfo.CategoryType == pubdef.TranslationTypeBottom {
for _, translationInfo := range translationByCategory[categoryInfo.Id] {
if translationInfo.IsFriendAds {
continue
} else if translationInfo.HtmlId > 0 {
title := strings.TrimSpace(translationInfo.Name)
if title != "" {
urlList = append(urlList, utils.JoinUrlPathKeepSlash(config.WebAddr, "/richtext?title="+url.QueryEscape(title)))
}
} else if len(translationInfo.Link) > 0 {
if !strings.HasPrefix(translationInfo.Link, "http://") && !strings.HasPrefix(translationInfo.Link, "https://") {
urlList = append(urlList, utils.JoinUrlPathAppendEnd(config.WebAddr, translationInfo.Link))
}
}
}
}
}
return urlList
}
+108
View File
@@ -0,0 +1,108 @@
package sitemap
import (
"testing"
"time"
"videoweb/pub/models/videocategorymod"
"videoweb/pub/models/videoinformationmod"
"go.mongodb.org/mongo-driver/bson"
)
func TestIsSitemapChannelPath(t *testing.T) {
if !isSitemapChannelPath("/drama", "/drama", "/drama-rank") {
t.Fatal("channel home should match")
}
if !isSitemapChannelPath("/drama/all", "/drama", "/drama-rank") {
t.Fatal("channel child path should match")
}
if !isSitemapChannelPath("/drama-rank", "/drama", "/drama-rank") {
t.Fatal("hyphen rank path should match")
}
if isSitemapChannelPath("/drama-rank", "/drama") {
t.Fatal("hyphen rank must not match prefix-only")
}
if isSitemapChannelPath("/video-rank", "/drama", "/drama-rank") {
t.Fatal("other channel rank should not match")
}
}
func TestBuildDetailURLs(t *testing.T) {
items := []idWithTime{
{Id: 53942, Slug: "sirozFs", LastMod: time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC)},
}
urls := buildDetailURLs("https://example.com/", "/video-detail", items, "weekly", 0.8)
if len(urls) != 1 {
t.Fatalf("buildDetailURLs() returned %d URLs, want 1", len(urls))
}
if want := "https://example.com/video-detail/sirozFs"; urls[0].Loc != want {
t.Fatalf("buildDetailURLs() loc = %q, want %q", urls[0].Loc, want)
}
fallback := buildDetailURLs("https://example.com/", "/video-detail", []idWithTime{{Id: 12}}, "weekly", 0.8)
if len(fallback) != 1 || fallback[0].Loc != "https://example.com/video-detail/12" {
t.Fatalf("empty slug should fall back to id, got %#v", fallback)
}
}
func TestBuildTagURLs(t *testing.T) {
slugURL := buildTagURLs("https://example.com/", "/video", []idWithTime{{Id: 9, Slug: "saduanfa"}}, "weekly", 0.6)
if len(slugURL) != 1 || slugURL[0].Loc != "https://example.com/video/t-saduanfa" {
t.Fatalf("slug tag loc = %#v", slugURL)
}
idURL := buildTagURLs("https://example.com/", "/drama", []idWithTime{{Id: 10142}}, "weekly", 0.6)
if len(idURL) != 1 || idURL[0].Loc != "https://example.com/drama/t-10142" {
t.Fatalf("id tag loc = %#v", idURL)
}
}
func TestBuildCategoryURLs(t *testing.T) {
slugURL := buildCategoryURLs("https://example.com/", "/video", []idWithTime{{Id: 37513, Slug: "hot-av"}}, "weekly", 0.6)
if len(slugURL) != 1 || slugURL[0].Loc != "https://example.com/video/c-hot-av" {
t.Fatalf("slug category loc = %#v", slugURL)
}
idURL := buildCategoryURLs("https://example.com/", "/drama", []idWithTime{{Id: 10001}}, "weekly", 0.6)
if len(idURL) != 1 || idURL[0].Loc != "https://example.com/drama/c-10001" {
t.Fatalf("id category loc = %#v", idURL)
}
}
func TestBuildPlayletSitemapURLs(t *testing.T) {
items := []idWithTime{{Id: 88, Slug: "chuanchengjiaofubei", LastMod: time.Date(2026, 9, 4, 0, 0, 0, 0, time.UTC)}}
detail := buildDetailURLs("https://example.com/", "/drama-detail", items, "weekly", 0.8)
if len(detail) != 1 || detail[0].Loc != "https://example.com/drama-detail/chuanchengjiaofubei" {
t.Fatalf("playlet detail loc = %#v", detail)
}
category := buildCategoryURLs("https://example.com/", "/drama", items, "weekly", 0.6)
if len(category) != 1 || category[0].Loc != "https://example.com/drama/c-chuanchengjiaofubei" {
t.Fatalf("playlet category loc = %#v", category)
}
idOnly := buildCategoryURLs("https://example.com/", "/video", []idWithTime{{Id: 37513}}, "weekly", 0.6)
if len(idOnly) != 1 || idOnly[0].Loc != "https://example.com/video/c-37513" {
t.Fatalf("category id fallback loc = %#v", idOnly)
}
tag := buildTagURLs("https://example.com/", "/drama", items, "weekly", 0.6)
if len(tag) != 1 || tag[0].Loc != "https://example.com/drama/t-chuanchengjiaofubei" {
t.Fatalf("playlet tag loc = %#v", tag)
}
}
func TestVideoCategoryContentFilter(t *testing.T) {
filter := videoCategoryContentFilter(&videocategorymod.VideoCategory{
Status: true,
Tags: []int64{11, 22},
})
tagsFilter, ok := filter[videoinformationmod.FN.VideoInformation.Tags].(bson.M)
if !ok {
t.Fatalf("tags filter = %#v, want bson.M", filter[videoinformationmod.FN.VideoInformation.Tags])
}
in, ok := tagsFilter["$in"].([]int64)
if !ok || len(in) != 2 || in[0] != 11 || in[1] != 22 {
t.Fatalf("tags $in = %#v, want [11 22]", tagsFilter["$in"])
}
emptyTags := videoCategoryContentFilter(&videocategorymod.VideoCategory{Status: true})
if _, exists := emptyTags[videoinformationmod.FN.VideoInformation.Tags]; exists {
t.Fatalf("empty tag category should not add tags filter: %#v", emptyTags)
}
}
+281
View File
@@ -0,0 +1,281 @@
package sitemap
import (
"context"
"fmt"
"net/url"
"strings"
"sync"
"time"
"videoweb/pub/common/utils"
"videoweb/pub/models/contenttagmod"
"videoweb/pub/models/playlethomeconfigmod"
"videoweb/pub/models/playletthememod"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
h5SortHot = "hot"
h5SortLatest = "latest"
h5TopicSort = "editorial"
h5RankPlaylet = "drama"
h5RankVideo = "video"
)
// getUrl 按 Next H5 路由生成 sitemap 分片:页面、短剧详情、视频详情。
func getUrl(sg *SitemapGenerator) {
webAddr := strings.TrimRight(strings.TrimSpace(sg.WebAddr), "/")
if webAddr == "" {
return
}
ctx := context.Background()
var (
wg sync.WaitGroup
videoEntries []videoEntry
videoTag map[int64]struct{}
playletItems []idWithTime
playletTagIDs map[int64]struct{}
themes []idWithTime
homeModules []idWithTime
)
wg.Add(4)
go func() {
defer wg.Done()
videoEntries, videoTag, _ = getVideoList(ctx)
}()
go func() {
defer wg.Done()
playletItems, _, playletTagIDs = getPlayletList(ctx)
}()
go func() {
defer wg.Done()
themes = getEnabledPlayletThemes(ctx)
}()
go func() {
defer wg.Done()
homeModules = getEnabledHomeModules(ctx)
}()
wg.Wait()
videoCatItems := getVideoCategoryList(ctx, videoTag, len(videoEntries) > 0)
videoTagItems := getContentTagList(ctx, videoTag, contenttagmod.ContentTagTypeVideo)
playletTagItems := getContentTagList(ctx, playletTagIDs, contenttagmod.ContentTagTypePlaylet)
now := time.Now()
pages := make([]URL, 0, 64)
seen := make(map[string]struct{})
add := func(dst *[]URL, loc, changeFreq string, priority float32, lastMod time.Time) {
loc = strings.TrimSpace(loc)
if loc == "" {
return
}
if _, ok := seen[loc]; ok {
return
}
seen[loc] = struct{}{}
if lastMod.IsZero() {
lastMod = now
}
*dst = append(*dst, URL{
Loc: loc,
LastMod: lastMod.Format(time.DateOnly),
ChangeFreq: changeFreq,
Priority: priority,
})
}
for _, spec := range h5StaticPages() {
add(&pages, h5Abs(webAddr, spec.path), spec.changeFreq, spec.priority, now)
}
for _, sort := range []string{h5SortHot, h5SortLatest} {
add(&pages, h5Abs(webAddr, "/short-drama", sort), "daily", 0.85, now)
add(&pages, h5Abs(webAddr, "/video", sort), "daily", 0.85, now)
for _, theme := range themes {
seg := detailSegment(theme.Slug, theme.Id)
if seg == "" || isReservedListSegment(seg) {
continue
}
add(&pages, h5Abs(webAddr, "/short-drama", sort, seg), "daily", 0.7, theme.LastMod)
}
for _, category := range videoCatItems {
seg := detailSegment(category.Slug, category.Id)
if seg == "" || isReservedListSegment(seg) {
continue
}
add(&pages, h5Abs(webAddr, "/video", sort, seg), "daily", 0.7, category.LastMod)
}
}
for _, tag := range playletTagItems {
seg := detailSegment(tag.Slug, tag.Id)
if seg == "" {
continue
}
add(&pages, h5Abs(webAddr, "/tag/playlet", seg), "weekly", 0.65, tag.LastMod)
}
for _, theme := range themes {
seg := detailSegment(theme.Slug, theme.Id)
if seg == "" {
continue
}
add(&pages, h5Abs(webAddr, "/topic/playlet", seg, h5TopicSort), "weekly", 0.7, theme.LastMod)
}
for _, tag := range videoTagItems {
seg := detailSegment(tag.Slug, tag.Id)
if seg == "" {
continue
}
add(&pages, h5Abs(webAddr, "/tag/video", seg), "weekly", 0.65, tag.LastMod)
}
for _, category := range videoCatItems {
seg := detailSegment(category.Slug, category.Id)
if seg == "" {
continue
}
add(&pages, h5Abs(webAddr, "/topic/video", seg, h5TopicSort), "weekly", 0.7, category.LastMod)
}
for _, module := range homeModules {
if module.Id <= 0 {
continue
}
add(&pages, h5Abs(webAddr, "/more", moreSegment(module.Slug, module.Id)), "daily", 0.65, module.LastMod)
}
if len(pages) > 0 {
_ = sg.writeSitemapFile("sitemap-pages.xml", pages)
}
playletURLs := make([]URL, 0, len(playletItems))
for _, item := range playletItems {
seg := detailSegment(item.Slug, item.Id)
if seg == "" || isReservedListSegment(seg) {
continue
}
add(&playletURLs, h5Abs(webAddr, "/short-drama", seg), "weekly", 0.8, item.LastMod)
}
if len(playletURLs) > 0 {
_ = sg.writeSitemapFiles("sitemap-playletdetail", playletURLs)
}
videoURLs := make([]URL, 0, len(videoEntries))
for _, item := range videoEntries {
seg := detailSegment(item.Slug, item.Id)
if seg == "" || isReservedListSegment(seg) {
continue
}
add(&videoURLs, h5Abs(webAddr, "/video", seg), "weekly", 0.8, item.LastMod)
}
if len(videoURLs) > 0 {
_ = sg.writeSitemapFiles("sitemap-videodetail", videoURLs)
}
}
type h5StaticPage struct {
path string
changeFreq string
priority float32
}
func h5StaticPages() []h5StaticPage {
return []h5StaticPage{
{path: "/", changeFreq: "daily", priority: 1},
{path: "/welfare", changeFreq: "weekly", priority: 0.6},
{path: "/nav", changeFreq: "weekly", priority: 0.7},
{path: "/way-home", changeFreq: "weekly", priority: 0.7},
{path: "/about", changeFreq: "monthly", priority: 0.4},
{path: "/terms", changeFreq: "monthly", priority: 0.3},
{path: "/privacy", changeFreq: "monthly", priority: 0.3},
{path: "/help", changeFreq: "monthly", priority: 0.3},
{path: "/ranking/" + h5RankPlaylet, changeFreq: "daily", priority: 0.8},
{path: "/ranking/" + h5RankVideo, changeFreq: "daily", priority: 0.8},
}
}
func isReservedListSegment(seg string) bool {
return seg == h5SortHot || seg == h5SortLatest || seg == h5RankPlaylet || seg == h5RankVideo
}
// moreSegment 首页更多页使用配置 slug。历史数据没有 slug 时使用 backfill 的字母标识。
func moreSegment(slug string, id int64) string {
if s := strings.TrimSpace(slug); s != "" {
return s
}
if id > 0 {
return fmt.Sprintf("home-config-%d", id)
}
return ""
}
// h5Abs 拼出与前端 encodeURIComponent 路径段一致的绝对地址。
func h5Abs(webAddr string, parts ...string) string {
segments := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.Trim(part, "/")
if part == "" {
continue
}
for _, seg := range strings.Split(part, "/") {
seg = strings.TrimSpace(seg)
if seg == "" {
continue
}
segments = append(segments, url.PathEscape(seg))
}
}
path := "/"
if len(segments) > 0 {
path = "/" + strings.Join(segments, "/")
}
return utils.JoinUrlPathKeepSlash(webAddr, path)
}
func getEnabledPlayletThemes(ctx context.Context) []idWithTime {
filter := bson.M{playletthememod.FN.PlayletTheme.Status: true}
pageSize := int64(1000)
items := make([]idWithTime, 0)
for i := int64(0); i < 100; i++ {
list, _ := playletthememod.FindManyPlayletTheme(ctx, filter, options.Find().
SetSort(bson.M{playletthememod.FN.PlayletTheme.Rank: -1}).
SetSkip(i*pageSize).
SetLimit(pageSize).
SetProjection(bson.M{
playletthememod.FN.PlayletTheme.Id: 1,
playletthememod.FN.PlayletTheme.Slug: 1,
playletthememod.FN.PlayletTheme.UpdatedAt: 1,
}))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
if len(list) < int(pageSize) {
break
}
}
return items
}
func getEnabledHomeModules(ctx context.Context) []idWithTime {
filter := bson.M{playlethomeconfigmod.FN.PlayletHomeConfig.Enabled: true}
list, _ := playlethomeconfigmod.FindManyPlayletHomeConfig(ctx, filter, options.Find().
SetProjection(bson.M{
playlethomeconfigmod.FN.PlayletHomeConfig.Id: 1,
playlethomeconfigmod.FN.PlayletHomeConfig.Slug: 1,
playlethomeconfigmod.FN.PlayletHomeConfig.UpdatedAt: 1,
}))
items := make([]idWithTime, 0, len(list))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
return items
}
+136
View File
@@ -0,0 +1,136 @@
package sitemap
import (
"context"
"videoweb/pub/models/contenttagmod"
"videoweb/pub/models/playletcategorymod"
"videoweb/pub/models/playletmod"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
// getPlayletList 拉取已上架短剧,并收集实际用到的分类/标签 ID。
func getPlayletList(ctx context.Context) ([]idWithTime, map[int64]struct{}, map[int64]struct{}) {
filter := bson.M{
playletmod.FN.Playlet.Status: playletmod.StatusUp,
}
pageSize := int64(1000)
items := make([]idWithTime, 0)
categoryIDs := make(map[int64]struct{})
tagIDs := make(map[int64]struct{})
for i := int64(0); i < 500; i++ {
skip := i * pageSize
list, _ := playletmod.FindManyPlaylet(ctx, filter, options.Find().
SetSort(bson.M{playletmod.FN.Playlet.Id: -1}).
SetSkip(skip).
SetLimit(pageSize).
SetProjection(bson.M{
playletmod.FN.Playlet.Id: 1,
playletmod.FN.Playlet.Slug: 1,
playletmod.FN.Playlet.UpdatedAt: 1,
playletmod.FN.Playlet.OnlineAt: 1,
playletmod.FN.Playlet.CategoryIds: 1,
playletmod.FN.Playlet.TagIds: 1,
}))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
lastMod := info.UpdatedAt
if lastMod.IsZero() {
lastMod = info.OnlineAt
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod})
for _, id := range info.CategoryIds {
if id > 0 {
categoryIDs[id] = struct{}{}
}
}
for _, id := range info.TagIds {
if id > 0 {
tagIDs[id] = struct{}{}
}
}
}
if len(list) < int(pageSize) {
break
}
}
return items, categoryIDs, tagIDs
}
// getPlayletCategoryList 只收录有上架短剧的分类,避免空列表页被抓取。
func getPlayletCategoryList(ctx context.Context, usedCategoryIDs map[int64]struct{}) []idWithTime {
if len(usedCategoryIDs) == 0 {
return nil
}
filter := bson.M{
playletcategorymod.FN.PlayletCategory.Status: true,
}
pageSize := int64(1000)
items := make([]idWithTime, 0)
for i := int64(0); i < 100; i++ {
skip := i * pageSize
list, _ := playletcategorymod.FindManyPlayletCategory(ctx, filter, options.Find().
SetSort(bson.M{playletcategorymod.FN.PlayletCategory.Rank: -1}).
SetSkip(skip).
SetLimit(pageSize).
SetProjection(bson.M{
playletcategorymod.FN.PlayletCategory.Id: 1,
playletcategorymod.FN.PlayletCategory.Slug: 1,
playletcategorymod.FN.PlayletCategory.UpdatedAt: 1,
}))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
if _, ok := usedCategoryIDs[info.Id]; !ok {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
if len(list) < int(pageSize) {
break
}
}
return items
}
// getContentTagList 只收录当前频道已绑定内容的统一标签。
func getContentTagList(ctx context.Context, usedTagIDs map[int64]struct{}, tagType contenttagmod.ContentTagType) []idWithTime {
if len(usedTagIDs) == 0 {
return nil
}
filter := bson.M{
contenttagmod.FN.ContentTag.Status: true,
contenttagmod.FN.ContentTag.Types: tagType,
}
pageSize := int64(1000)
items := make([]idWithTime, 0)
for i := int64(0); i < 100; i++ {
skip := i * pageSize
list, _ := contenttagmod.FindManyContentTag(ctx, filter, options.Find().
SetSort(bson.M{contenttagmod.FN.ContentTag.Rank: -1}).
SetSkip(skip).
SetLimit(pageSize).
SetProjection(bson.M{
contenttagmod.FN.ContentTag.Id: 1,
contenttagmod.FN.ContentTag.Slug: 1,
contenttagmod.FN.ContentTag.UpdatedAt: 1,
}))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
if _, ok := usedTagIDs[info.Id]; !ok {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
if len(list) < int(pageSize) {
break
}
}
return items
}
+245
View File
@@ -0,0 +1,245 @@
package sitemap
import (
"encoding/xml"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
const (
// 浏览器用 XSLT 把 loc 渲成可点击链接;搜索引擎仍按 XML 解析
stylesheetIndex = `<?xml-stylesheet type="text/xsl" href="/static/sitemapindex.xsl"?>` + "\n"
stylesheetURLSet = `<?xml-stylesheet type="text/xsl" href="/static/sitemap.xsl"?>` + "\n"
// defaultMaxURLsPerFile 单文件 URL 上限;Google 上限 50000,留余量避免体积过大。
defaultMaxURLsPerFile = 40000
)
// URL 单个URL条目
type URL struct {
Loc string `xml:"loc"`
LastMod string `xml:"lastmod"`
ChangeFreq string `xml:"changefreq"` // always/hourly/daily/weekly/monthly/yearly/never
Priority float32 `xml:"priority"` // 0.0 - 1.0
}
// Sitemap 站点地图文件
type Sitemap struct {
XMLName xml.Name `xml:"urlset"`
Xmlns string `xml:"xmlns,attr"`
URLs []URL `xml:"url"`
}
// SitemapIndex 站点地图索引
type SitemapIndex struct {
XMLName xml.Name `xml:"sitemapindex"`
Xmlns string `xml:"xmlns,attr"`
Sitemaps []SitemapIndexEntry `xml:"sitemap"`
}
// SitemapIndexEntry 站点地图索引条目
type SitemapIndexEntry struct {
Loc string `xml:"loc"`
LastMod string `xml:"lastmod"`
}
// SitemapGenerator 站点地图生成器
type SitemapGenerator struct {
MaxURLsPerFile int // 每个sitemap文件最大URL数量
OutputDir string // 分片输出目录(/sitemaps)
IndexFile string // 索引文件路径;空则写到 OutputDir/sitemap.xml
WebAddr string // 网站地址
SitemapFiles []string // 生成的sitemap文件列表
}
// NewSitemapGenerator 创建新的站点地图生成器
func NewSitemapGenerator(webAddr, outputDir string) *SitemapGenerator {
return &SitemapGenerator{
MaxURLsPerFile: defaultMaxURLsPerFile,
OutputDir: outputDir,
WebAddr: webAddr,
SitemapFiles: make([]string, 0),
}
}
func (sg *SitemapGenerator) maxURLsPerFile() int {
if sg.MaxURLsPerFile > 0 {
return sg.MaxURLsPerFile
}
return defaultMaxURLsPerFile
}
// Generate 生成站点地图
func (sg *SitemapGenerator) Generate() error {
// 清空之前的文件列表
sg.SitemapFiles = make([]string, 0)
// 创建输出目录
if err := os.MkdirAll(sg.OutputDir, 0755); err != nil {
return err
}
deprecatedFiles := []string{
"sitemap-createpostlist.xml",
"sitemap-postlist.xml",
"sitemap-posttag.xml",
"sitemap-createvideolist.xml",
"sitemap-tag.xml",
"sitemap-translation.xml",
"sitemap-actress.xml",
"sitemap-videocategory.xml",
"sitemap-playletcategory.xml",
"sitemap-videotag.xml",
"sitemap-playlettag.xml",
"sitemap-archives.xml",
"sitemap-videos.xml",
}
for _, filename := range deprecatedFiles {
if err := os.Remove(filepath.Join(sg.OutputDir, filename)); err != nil && !os.IsNotExist(err) {
return err
}
}
sg.cleanupSitemapShards("sitemap-archives", 0)
sg.cleanupSitemapShards("sitemap-videos", 0)
if sg.IndexFile != "" && filepath.Clean(sg.IndexFile) != filepath.Clean(filepath.Join(sg.OutputDir, "sitemap.xml")) {
if err := os.Remove(filepath.Join(sg.OutputDir, "sitemap.xml")); err != nil && !os.IsNotExist(err) {
return err
}
}
getUrl(sg)
// 生成sitemap索引文件
return sg.generateSitemapIndex()
}
// writeSitemapFile 写入sitemap文件
func (sg *SitemapGenerator) writeSitemapFile(filename string, urls []URL) error {
sitemap := Sitemap{
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
URLs: urls,
}
filePath := filepath.Join(sg.OutputDir, filename)
file, err := os.Create(filePath)
if err != nil {
return err
}
defer file.Close()
// 添加XML头
file.WriteString(xml.Header)
file.WriteString(stylesheetURLSet)
encoder := xml.NewEncoder(file)
encoder.Indent("", " ")
if err := encoder.Encode(sitemap); err != nil {
return err
}
// 记录生成的文件
sg.SitemapFiles = append(sg.SitemapFiles, filename)
return nil
}
// writeSitemapFiles 写入 sitemap 分片;超过上限时拆为 baseName-0.xml、baseName-1.xml ...
func (sg *SitemapGenerator) writeSitemapFiles(baseName string, urls []URL) error {
baseName = strings.TrimSuffix(strings.TrimSpace(baseName), ".xml")
if baseName == "" {
return nil
}
if len(urls) == 0 {
sg.cleanupSitemapShards(baseName, 0)
return nil
}
limit := sg.maxURLsPerFile()
shardCount := (len(urls) + limit - 1) / limit
if shardCount == 1 {
sg.cleanupSitemapShards(baseName, 1)
return sg.writeSitemapFile(baseName+".xml", urls)
}
sg.cleanupSitemapShards(baseName, shardCount)
for i := 0; i < shardCount; i++ {
start := i * limit
end := start + limit
if end > len(urls) {
end = len(urls)
}
filename := fmt.Sprintf("%s-%d.xml", baseName, i)
if err := sg.writeSitemapFile(filename, urls[start:end]); err != nil {
return err
}
}
return nil
}
// cleanupSitemapShards 清理旧分片:shardCount=1 保留 baseName.xml;>1 保留 baseName-0..N-1.xml。
func (sg *SitemapGenerator) cleanupSitemapShards(baseName string, shardCount int) {
entries, err := os.ReadDir(sg.OutputDir)
if err != nil {
return
}
prefix := baseName + "-"
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ".xml") {
idxText := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".xml")
idx, err := strconv.Atoi(idxText)
if err != nil {
continue
}
if shardCount <= 1 || idx >= shardCount {
_ = os.Remove(filepath.Join(sg.OutputDir, name))
}
continue
}
if shardCount > 1 && name == baseName+".xml" {
_ = os.Remove(filepath.Join(sg.OutputDir, name))
}
}
}
// generateSitemapIndex 生成sitemap索引文件
func (sg *SitemapGenerator) generateSitemapIndex() error {
index := SitemapIndex{
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
Sitemaps: make([]SitemapIndexEntry, 0, len(sg.SitemapFiles)),
}
now := time.Now().Format(time.DateOnly)
webAddr := strings.TrimRight(strings.TrimSpace(sg.WebAddr), "/")
for _, filename := range sg.SitemapFiles {
index.Sitemaps = append(index.Sitemaps, SitemapIndexEntry{
Loc: fmt.Sprintf("%s/sitemaps/%s", webAddr, filename),
LastMod: now,
})
}
filePath := sg.IndexFile
if strings.TrimSpace(filePath) == "" {
filePath = filepath.Join(sg.OutputDir, "sitemap.xml")
}
if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
return err
}
file, err := os.Create(filePath)
if err != nil {
return err
}
defer file.Close()
// 添加XML头
file.WriteString(xml.Header)
file.WriteString(stylesheetIndex)
encoder := xml.NewEncoder(file)
encoder.Indent("", " ")
return encoder.Encode(index)
}
+180
View File
@@ -0,0 +1,180 @@
package sitemap
import (
"bytes"
"encoding/xml"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"unicode/utf8"
"videoweb/pub/common/netutil"
"videoweb/pub/common/utils"
)
const (
videoSitemapShardPrefix = "sitemap-videos-"
videoSitemapDefaultThumb = "/static/images/og-default.jpg"
videoSitemapDefaultDesc = "高清成人视频在线观看。"
videoSitemapEmbedPathFmt = "/video/embed?id=%d"
)
type videoEntry struct {
Id int64
Slug string
Title string
LastMod time.Time
UnencryptedPreMp4 string
ContentLoc string
}
// prepareVideoSitemapEntries 为全部视频补齐 sitemap 字段;有 UnencryptedPreMp4 时额外设置 content_loc。
func prepareVideoSitemapEntries(entries []videoEntry, cdn string) []videoEntry {
cdn = strings.TrimSpace(cdn)
out := make([]videoEntry, 0, len(entries))
for _, e := range entries {
if e.Id <= 0 {
continue
}
if cdn != "" {
if preMp4 := strings.TrimSpace(e.UnencryptedPreMp4); preMp4 != "" {
if contentLoc := netutil.MergeURL(cdn, preMp4); contentLoc != "" {
e.ContentLoc = contentLoc
}
}
}
out = append(out, e)
}
return out
}
func (sg *SitemapGenerator) writeVideoSitemapFiles(entries []videoEntry) error {
if len(entries) == 0 {
sg.cleanupVideoSitemapShards(0)
return nil
}
limit := sg.maxURLsPerFile()
shardCount := (len(entries) + limit - 1) / limit
sg.cleanupVideoSitemapShards(shardCount)
thumb := strings.TrimRight(sg.WebAddr, "/") + videoSitemapDefaultThumb
for i := 0; i < len(entries); i += limit {
end := i + limit
if end > len(entries) {
end = len(entries)
}
name := fmt.Sprintf("%s%d.xml", videoSitemapShardPrefix, i/limit)
if err := sg.writeOneVideoSitemap(name, entries[i:end], thumb); err != nil {
return err
}
sg.SitemapFiles = append(sg.SitemapFiles, name)
}
return nil
}
func (sg *SitemapGenerator) cleanupVideoSitemapShards(shardCount int) {
entries, err := os.ReadDir(sg.OutputDir)
if err != nil {
return
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if !strings.HasPrefix(name, videoSitemapShardPrefix) || !strings.HasSuffix(name, ".xml") {
continue
}
idxText := strings.TrimSuffix(strings.TrimPrefix(name, videoSitemapShardPrefix), ".xml")
idx, err := strconv.Atoi(idxText)
if err != nil || shardCount <= 0 || idx >= shardCount {
_ = os.Remove(filepath.Join(sg.OutputDir, name))
}
}
}
func (sg *SitemapGenerator) writeOneVideoSitemap(filename string, entries []videoEntry, thumb string) error {
var b bytes.Buffer
b.WriteString(xml.Header)
b.WriteString(stylesheetURLSet)
b.WriteString(`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"` + "\n")
b.WriteString(` xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">` + "\n")
for _, e := range entries {
if e.Id <= 0 {
continue
}
loc := videoEntryLoc(sg.WebAddr, e)
if loc == "" {
continue
}
playerLoc := videoEntryPlayerLoc(sg.WebAddr, e.Id)
title := xmlEscapeVideoSitemap(videoEntryTitle(e))
desc := xmlEscapeVideoSitemap(videoSitemapDefaultDesc)
lastMod := videoEntryLastMod(e)
b.WriteString(" <url>\n")
fmt.Fprintf(&b, " <loc>%s</loc>\n", xmlEscapeVideoSitemap(loc))
fmt.Fprintf(&b, " <lastmod>%s</lastmod>\n", xmlEscapeVideoSitemap(lastMod))
b.WriteString(" <changefreq>weekly</changefreq>\n")
b.WriteString(" <priority>0.8</priority>\n")
b.WriteString(" <video:video>\n")
fmt.Fprintf(&b, " <video:thumbnail_loc>%s</video:thumbnail_loc>\n", xmlEscapeVideoSitemap(thumb))
fmt.Fprintf(&b, " <video:title>%s</video:title>\n", title)
fmt.Fprintf(&b, " <video:description>%s</video:description>\n", desc)
if contentLoc := strings.TrimSpace(e.ContentLoc); contentLoc != "" {
fmt.Fprintf(&b, " <video:content_loc>%s</video:content_loc>\n", xmlEscapeVideoSitemap(contentLoc))
}
fmt.Fprintf(&b, " <video:player_loc>%s</video:player_loc>\n", xmlEscapeVideoSitemap(playerLoc))
b.WriteString(" <video:family_friendly>no</video:family_friendly>\n")
b.WriteString(" </video:video>\n")
b.WriteString(" </url>\n")
}
b.WriteString("</urlset>\n")
path := filepath.Join(sg.OutputDir, filename)
return os.WriteFile(path, b.Bytes(), 0644)
}
func videoEntryLoc(webAddr string, e videoEntry) string {
seg := detailSegment(e.Slug, e.Id)
if seg == "" {
return ""
}
return utils.JoinUrlPathKeepSlash(webAddr, "/video-detail/"+seg)
}
func videoEntryPlayerLoc(webAddr string, id int64) string {
return utils.JoinUrlPathKeepSlash(webAddr, fmt.Sprintf(videoSitemapEmbedPathFmt, id))
}
func videoEntryTitle(e videoEntry) string {
if title := strings.TrimSpace(e.Title); title != "" {
return truncateRunesVideoSitemap(title, 100)
}
return fmt.Sprintf("视频 %d", e.Id)
}
func videoEntryLastMod(e videoEntry) string {
if e.LastMod.IsZero() {
return time.Now().Format(time.DateOnly)
}
return e.LastMod.Format(time.DateOnly)
}
func xmlEscapeVideoSitemap(s string) string {
var buf bytes.Buffer
_ = xml.EscapeText(&buf, []byte(s))
return buf.String()
}
func truncateRunesVideoSitemap(s string, max int) string {
if max <= 0 || utf8.RuneCountInString(s) <= max {
return s
}
runes := []rune(s)
return string(runes[:max])
}
+101
View File
@@ -0,0 +1,101 @@
package seo
import (
"context"
"path/filepath"
"strings"
"sync"
"time"
"videoweb/app/service/configser"
"videoweb/app/skd/job/dhsitemap"
"videoweb/app/skd/job/llmstxt"
"videoweb/app/skd/job/rss"
"videoweb/app/skd/job/sitemap"
"videoweb/pub/common/log"
"videoweb/pub/seo/sitemapdirty"
"github.com/robfig/cron/v3"
)
const (
// h5OutputDir 站点地图写在 h5 根目录,不进 public。
// 样式表仍由前端 public/static 提供,XML 里引用 /static/sitemap*.xsl。
h5OutputDir = "../h5"
pageTemplateOutputDir = "../h5/templates/page"
)
var seoCron *cron.Cron
func init() {
seoCron = cron.New(cron.WithSeconds())
}
var (
dirtyMu sync.Mutex
// generatedDirtyAt 最近一次 sitemap 生成所对应的内容变更时间
generatedDirtyAt time.Time
)
// Start 启动 SEO 静态资源生成(sitemap / RSS / llms.txt / dhsitemap)定时任务。
// 页面级搜索引擎推送由 web 进程的 seopush 任务负责,不在这里处理。
func Start() {
generatedDirtyAt = sitemapdirty.LastDirtyAt(context.Background())
go generateSitemapFiles(context.Background())
go dhsitemap.GenerateDHSitemap()
go llmstxt.GenerateLLMSTxt(pageTemplateOutputDir)
_, _ = seoCron.AddFunc("0 0 */6 * * ?", func() {
generateSitemapFiles(context.Background())
dhsitemap.GenerateDHSitemap()
llmstxt.GenerateLLMSTxt(pageTemplateOutputDir)
})
// 短剧/视频等发布发生在 web 进程,这里检查它留下的变更标记,有变更才重新生成
_, _ = seoCron.AddFunc("0 */2 * * * ?", func() {
regenerateSitemapIfDirty()
})
seoCron.Start()
log.Info("seo skd started")
}
func generateSitemapFiles(ctx context.Context) {
config, serr := configser.GetSystemConfig(ctx)
if serr != nil || config == nil {
log.Error("gen sitemap GetSystemConfig failed", log.Any("err", serr))
return
}
webAddr := strings.TrimSpace(config.WebAddr)
if webAddr == "" {
log.Error("gen sitemap WebAddr empty")
return
}
generator := sitemap.NewSitemapGenerator(webAddr, filepath.Join(h5OutputDir, "sitemaps"))
generator.IndexFile = filepath.Join(h5OutputDir, "sitemap.xml")
if err := generator.Generate(); err != nil {
log.Error("gen sitemap.xml err", log.E(err))
return
}
rss.GenerateRSS(ctx, pageTemplateOutputDir)
}
func regenerateSitemapIfDirty() {
if !dirtyMu.TryLock() {
return
}
defer dirtyMu.Unlock()
dirtyAt := sitemapdirty.LastDirtyAt(context.Background())
if dirtyAt.IsZero() || !dirtyAt.After(generatedDirtyAt) {
return
}
log.Info("seo: content changed, regenerate sitemap", log.Any("dirtyAt", dirtyAt))
generateSitemapFiles(context.Background())
dhsitemap.GenerateDHSitemap()
llmstxt.GenerateLLMSTxt(pageTemplateOutputDir)
generatedDirtyAt = dirtyAt
}
func Stop() {
if seoCron != nil {
seoCron.Stop()
}
}