rootandClaude Opus 4.7
20b5e05c78
前端 /play/[slug]/ 会 301 到 /play/[slug]/[chapterId]/，sitemap 直接按首集 ID 拼好地址，避免爬虫走一次 301 浪费抓取预算。原创短剧要求 watermarkVideoUrl 非空。 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
423 lines
12 KiB
Go
423 lines
12 KiB
Go
package sitemap
|
|
|
|
import (
|
|
"context"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
"videoweb/pub/common/utils"
|
|
"videoweb/pub/models/playletcategorymod"
|
|
"videoweb/pub/models/playletchaptermod"
|
|
"videoweb/pub/models/playletmod"
|
|
"videoweb/pub/models/playlettagmod"
|
|
"videoweb/pub/models/playletthememod"
|
|
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
type idWithTime struct {
|
|
Id int64
|
|
Slug string
|
|
LastMod time.Time
|
|
IsOriginal bool
|
|
}
|
|
|
|
type staticPageEntry struct {
|
|
path string
|
|
changeFreq string
|
|
priority float32
|
|
}
|
|
|
|
func buildStaticURLs(webAddr string, entries []staticPageEntry) []URL {
|
|
urls := make([]URL, 0, len(entries))
|
|
now := time.Now().Format(time.DateOnly)
|
|
for _, entry := range entries {
|
|
urls = append(urls, URL{
|
|
Loc: utils.JoinUrlPathKeepSlash(webAddr, entry.path),
|
|
LastMod: now,
|
|
ChangeFreq: entry.changeFreq,
|
|
Priority: entry.priority,
|
|
})
|
|
}
|
|
return urls
|
|
}
|
|
|
|
func entityPath(prefix, slug string, id int64) string {
|
|
if segment := utils.PathSegment(slug, id); segment != "" {
|
|
return strings.TrimSuffix(prefix, "/") + "/" + segment + "/"
|
|
}
|
|
if strings.HasSuffix(prefix, "/") {
|
|
return prefix
|
|
}
|
|
return prefix + "/"
|
|
}
|
|
|
|
func hgdjStaticPages() []staticPageEntry {
|
|
return []staticPageEntry{
|
|
{path: "/", changeFreq: "daily", priority: 1.0},
|
|
{path: "/list/ai-short-play/", changeFreq: "daily", priority: 0.9},
|
|
{path: "/list/ai-short-play/popular/", changeFreq: "daily", priority: 0.8},
|
|
{path: "/list/ai-short-play/original/", changeFreq: "daily", priority: 0.8},
|
|
{path: "/series/", changeFreq: "daily", priority: 0.9},
|
|
{path: "/series/popular/", changeFreq: "daily", priority: 0.8},
|
|
{path: "/series/original/", changeFreq: "daily", priority: 0.8},
|
|
{path: "/rank/", changeFreq: "daily", priority: 0.8},
|
|
{path: "/today-update/", changeFreq: "daily", priority: 0.8},
|
|
{path: "/topic/", changeFreq: "daily", priority: 0.8},
|
|
{path: "/application/", changeFreq: "weekly", priority: 0.6},
|
|
{path: "/home-road/", changeFreq: "weekly", priority: 0.6},
|
|
{path: "/download-app/", changeFreq: "monthly", priority: 0.5},
|
|
{path: "/nav/", changeFreq: "monthly", priority: 0.5},
|
|
{path: "/about-us/", changeFreq: "yearly", priority: 0.3},
|
|
{path: "/help-center/", changeFreq: "yearly", priority: 0.3},
|
|
}
|
|
}
|
|
|
|
// getURL 并发拉取短剧数据并写入 sitemap 分片，URL 规则对齐 jha-1043-hgdj-web。
|
|
func getURL(sg *SitemapGenerator) {
|
|
webAddr := strings.TrimSpace(sg.WebAddr)
|
|
if webAddr == "" {
|
|
return
|
|
}
|
|
|
|
ctx := context.Background()
|
|
var (
|
|
wg sync.WaitGroup
|
|
listed listedPlaylets
|
|
themeItems []idWithTime
|
|
tagItems []idWithTime
|
|
categoryItems []idWithTime
|
|
)
|
|
|
|
wg.Add(4)
|
|
go func() {
|
|
defer wg.Done()
|
|
listed = getPlayletList(ctx)
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
themeItems = getThemeList(ctx)
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
tagItems = getTagList(ctx)
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
categoryItems = getCategoryList(ctx)
|
|
}()
|
|
wg.Wait()
|
|
|
|
// 专题/标签没有上架短剧时不写入，避免空列表页被收录。
|
|
// 分类页无绑定短剧时会回退展示全部上架短剧，因此启用分类都写入。
|
|
themeItems = keepWithListedPlaylets(themeItems, listed.themeIDs)
|
|
tagItems = keepWithListedPlaylets(tagItems, listed.tagIDs)
|
|
|
|
pageURLs := buildStaticURLs(webAddr, hgdjStaticPages())
|
|
for _, item := range themeItems {
|
|
pageURLs = append(pageURLs, URL{
|
|
Loc: utils.JoinUrlPathKeepSlash(webAddr, entityPath("/topic", item.Slug, item.Id)),
|
|
LastMod: formatLastMod(item.LastMod),
|
|
ChangeFreq: "daily",
|
|
Priority: 0.8,
|
|
})
|
|
}
|
|
for _, item := range categoryItems {
|
|
pageURLs = append(pageURLs, URL{
|
|
Loc: utils.JoinUrlPathKeepSlash(webAddr, entityPath("/category", item.Slug, item.Id)),
|
|
LastMod: formatLastMod(item.LastMod),
|
|
ChangeFreq: "daily",
|
|
Priority: 0.8,
|
|
})
|
|
}
|
|
for _, item := range tagItems {
|
|
pageURLs = append(pageURLs, URL{
|
|
Loc: utils.JoinUrlPathKeepSlash(webAddr, entityPath("/tag", item.Slug, item.Id)),
|
|
LastMod: formatLastMod(item.LastMod),
|
|
ChangeFreq: "weekly",
|
|
Priority: 0.6,
|
|
})
|
|
}
|
|
if len(pageURLs) > 0 {
|
|
_ = sg.writeSitemapFile("sitemap-pages.xml", pageURLs)
|
|
}
|
|
|
|
if urls := buildPlayletPathURLs(webAddr, listed.items, "/detail", "weekly", 0.7); len(urls) > 0 {
|
|
_ = sg.writeSitemapFiles("sitemap-playletdetail", urls)
|
|
}
|
|
// /play 页面需带上默认剧集 ID，否则前端会 301 到 /play/[slug]/[chapterId]/，浪费爬虫抓取预算。
|
|
chapterMap := getPlayletFirstChapterMap(ctx, listed.items)
|
|
if urls := buildPlayletPlayURLs(webAddr, listed.items, chapterMap, "weekly", 0.6); len(urls) > 0 {
|
|
_ = sg.writeSitemapFiles("sitemap-playletplay", urls)
|
|
}
|
|
}
|
|
|
|
type listedPlaylets struct {
|
|
items []idWithTime
|
|
themeIDs map[int64]struct{}
|
|
tagIDs map[int64]struct{}
|
|
categoryIDs map[int64]struct{}
|
|
}
|
|
|
|
func keepWithListedPlaylets(items []idWithTime, listed map[int64]struct{}) []idWithTime {
|
|
if len(items) == 0 || len(listed) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]idWithTime, 0, len(items))
|
|
for _, item := range items {
|
|
if _, ok := listed[item.Id]; ok {
|
|
out = append(out, item)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func addListedIDs(dst map[int64]struct{}, ids []int64) {
|
|
for _, id := range ids {
|
|
if id > 0 {
|
|
dst[id] = struct{}{}
|
|
}
|
|
}
|
|
}
|
|
|
|
func buildPlayletPathURLs(webAddr string, items []idWithTime, prefix string, changeFreq string, priority float32) []URL {
|
|
urls := make([]URL, 0, len(items))
|
|
for _, item := range items {
|
|
lastMod := formatLastMod(item.LastMod)
|
|
if lastMod == "" {
|
|
lastMod = time.Now().Format(time.DateOnly)
|
|
}
|
|
urls = append(urls, URL{
|
|
Loc: utils.JoinUrlPathKeepSlash(webAddr, entityPath(prefix, item.Slug, item.Id)),
|
|
LastMod: lastMod,
|
|
ChangeFreq: changeFreq,
|
|
Priority: priority,
|
|
})
|
|
}
|
|
return urls
|
|
}
|
|
|
|
// buildPlayletPlayURLs 构建播放页 sitemap URL：有默认剧集时直接拼成 /play/[slug]/[chapterId]/，与前端规范地址一致，
|
|
// 避免爬虫经由 301 跳到带剧集 ID 的地址。查不到剧集的短剧跳过，不输出会 301 或 404 的 URL。
|
|
func buildPlayletPlayURLs(webAddr string, items []idWithTime, chapterMap map[int64]int64, changeFreq string, priority float32) []URL {
|
|
urls := make([]URL, 0, len(items))
|
|
for _, item := range items {
|
|
chapterId, ok := chapterMap[item.Id]
|
|
if !ok || chapterId <= 0 {
|
|
continue
|
|
}
|
|
segment := utils.PathSegment(item.Slug, item.Id)
|
|
if segment == "" {
|
|
continue
|
|
}
|
|
path := "/play/" + segment + "/" + strconv.FormatInt(chapterId, 10) + "/"
|
|
lastMod := formatLastMod(item.LastMod)
|
|
if lastMod == "" {
|
|
lastMod = time.Now().Format(time.DateOnly)
|
|
}
|
|
urls = append(urls, URL{
|
|
Loc: utils.JoinUrlPathKeepSlash(webAddr, path),
|
|
LastMod: lastMod,
|
|
ChangeFreq: changeFreq,
|
|
Priority: priority,
|
|
})
|
|
}
|
|
return urls
|
|
}
|
|
|
|
// getPlayletFirstChapterMap 批量查询每部短剧的默认（首集）剧集 ID，与前端 /play/[slug]/ 的 301 目标保持一致：
|
|
// 按 currentEpisode 升序取首条；原创短剧要求 watermarkVideoUrl 非空（未处理好的集数不露出，避免前端再次跳转）。
|
|
func getPlayletFirstChapterMap(ctx context.Context, items []idWithTime) map[int64]int64 {
|
|
result := make(map[int64]int64, len(items))
|
|
if len(items) == 0 {
|
|
return result
|
|
}
|
|
ids := make([]int64, 0, len(items))
|
|
originalSet := make(map[int64]struct{}, len(items))
|
|
for _, item := range items {
|
|
if item.Id <= 0 {
|
|
continue
|
|
}
|
|
ids = append(ids, item.Id)
|
|
if item.IsOriginal {
|
|
originalSet[item.Id] = struct{}{}
|
|
}
|
|
}
|
|
const batchSize = 200
|
|
for start := 0; start < len(ids); start += batchSize {
|
|
end := start + batchSize
|
|
if end > len(ids) {
|
|
end = len(ids)
|
|
}
|
|
batch := ids[start:end]
|
|
list, err := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{
|
|
playletchaptermod.FN.PlayletChapter.PlayletId: bson.M{"$in": batch},
|
|
}, options.Find().
|
|
SetProjection(bson.M{
|
|
playletchaptermod.FN.PlayletChapter.Id: 1,
|
|
playletchaptermod.FN.PlayletChapter.PlayletId: 1,
|
|
playletchaptermod.FN.PlayletChapter.CurrentEpisode: 1,
|
|
playletchaptermod.FN.PlayletChapter.WatermarkVideoUrl: 1,
|
|
}).
|
|
SetSort(bson.D{
|
|
{Key: playletchaptermod.FN.PlayletChapter.PlayletId, Value: 1},
|
|
{Key: playletchaptermod.FN.PlayletChapter.CurrentEpisode, Value: 1},
|
|
{Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1},
|
|
}))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, chapter := range list {
|
|
if chapter == nil || chapter.PlayletId <= 0 || chapter.Id <= 0 {
|
|
continue
|
|
}
|
|
if _, ok := result[chapter.PlayletId]; ok {
|
|
continue
|
|
}
|
|
if _, isOriginal := originalSet[chapter.PlayletId]; isOriginal {
|
|
if strings.TrimSpace(chapter.WatermarkVideoUrl) == "" {
|
|
continue
|
|
}
|
|
}
|
|
result[chapter.PlayletId] = chapter.Id
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func formatLastMod(t time.Time) string {
|
|
if t.IsZero() {
|
|
return ""
|
|
}
|
|
return t.Format(time.DateOnly)
|
|
}
|
|
|
|
func getPlayletList(ctx context.Context) listedPlaylets {
|
|
result := listedPlaylets{
|
|
items: make([]idWithTime, 0),
|
|
themeIDs: make(map[int64]struct{}),
|
|
tagIDs: make(map[int64]struct{}),
|
|
categoryIDs: make(map[int64]struct{}),
|
|
}
|
|
filter := bson.M{
|
|
playletmod.FN.Playlet.Status: playletmod.StatusUp,
|
|
}
|
|
pageSize := int64(1000)
|
|
for i := int64(0); i < 500; i++ {
|
|
skip := i * pageSize
|
|
list, _ := playletmod.FindManyPlaylet(ctx, filter, options.Find().
|
|
SetSort(bson.M{playletmod.FN.Playlet.Id: -1}).
|
|
SetSkip(skip).
|
|
SetLimit(pageSize).
|
|
SetProjection(bson.M{
|
|
playletmod.FN.Playlet.Id: 1,
|
|
playletmod.FN.Playlet.Slug: 1,
|
|
playletmod.FN.Playlet.UpdatedAt: 1,
|
|
playletmod.FN.Playlet.OnlineAt: 1,
|
|
playletmod.FN.Playlet.IsOriginal: 1,
|
|
playletmod.FN.Playlet.ThemeIds: 1,
|
|
playletmod.FN.Playlet.TagIds: 1,
|
|
playletmod.FN.Playlet.CategoryIds: 1,
|
|
}))
|
|
for _, info := range list {
|
|
if info == nil || info.Id <= 0 {
|
|
continue
|
|
}
|
|
lastMod := info.UpdatedAt
|
|
if lastMod.IsZero() {
|
|
lastMod = info.OnlineAt
|
|
}
|
|
result.items = append(result.items, idWithTime{
|
|
Id: info.Id,
|
|
Slug: info.Slug,
|
|
LastMod: lastMod,
|
|
IsOriginal: info.IsOriginal,
|
|
})
|
|
addListedIDs(result.themeIDs, info.ThemeIds)
|
|
addListedIDs(result.tagIDs, info.TagIds)
|
|
addListedIDs(result.categoryIDs, info.CategoryIds)
|
|
}
|
|
if len(list) < int(pageSize) {
|
|
break
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func getThemeList(ctx context.Context) []idWithTime {
|
|
filter := bson.M{
|
|
playletthememod.FN.PlayletTheme.Status: true,
|
|
}
|
|
pageSize := int64(1000)
|
|
items := make([]idWithTime, 0)
|
|
for i := int64(0); i < 100; i++ {
|
|
skip := i * pageSize
|
|
list, _ := playletthememod.FindManyPlayletTheme(ctx, filter, options.Find().
|
|
SetSort(bson.M{playletthememod.FN.PlayletTheme.Rank: -1}).
|
|
SetSkip(skip).
|
|
SetLimit(pageSize))
|
|
for _, info := range list {
|
|
if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" {
|
|
continue
|
|
}
|
|
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
|
}
|
|
if len(list) < int(pageSize) {
|
|
break
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
func getTagList(ctx context.Context) []idWithTime {
|
|
filter := bson.M{
|
|
playlettagmod.FN.PlayletTag.Status: true,
|
|
}
|
|
pageSize := int64(1000)
|
|
items := make([]idWithTime, 0)
|
|
for i := int64(0); i < 100; i++ {
|
|
skip := i * pageSize
|
|
list, _ := playlettagmod.FindManyPlayletTag(ctx, filter, options.Find().
|
|
SetSort(bson.M{playlettagmod.FN.PlayletTag.Rank: -1}).
|
|
SetSkip(skip).
|
|
SetLimit(pageSize))
|
|
for _, info := range list {
|
|
if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" {
|
|
continue
|
|
}
|
|
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
|
}
|
|
if len(list) < int(pageSize) {
|
|
break
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
func getCategoryList(ctx context.Context) []idWithTime {
|
|
filter := bson.M{
|
|
playletcategorymod.FN.PlayletCategory.Status: true,
|
|
}
|
|
pageSize := int64(1000)
|
|
items := make([]idWithTime, 0)
|
|
for i := int64(0); i < 100; i++ {
|
|
skip := i * pageSize
|
|
list, _ := playletcategorymod.FindManyPlayletCategory(ctx, filter, options.Find().
|
|
SetSort(bson.M{playletcategorymod.FN.PlayletCategory.Rank: -1}).
|
|
SetSkip(skip).
|
|
SetLimit(pageSize))
|
|
for _, info := range list {
|
|
if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" {
|
|
continue
|
|
}
|
|
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
|
}
|
|
if len(list) < int(pageSize) {
|
|
break
|
|
}
|
|
}
|
|
return items
|
|
}
|