diff --git a/app/skd/job/sitemap/data.go b/app/skd/job/sitemap/data.go index 04b7839..dc66397 100644 --- a/app/skd/job/sitemap/data.go +++ b/app/skd/job/sitemap/data.go @@ -2,11 +2,13 @@ package sitemap import ( "context" + "strconv" "strings" "sync" "time" "videoweb/pub/common/utils" "videoweb/pub/models/playletcategorymod" + "videoweb/pub/models/playletchaptermod" "videoweb/pub/models/playletmod" "videoweb/pub/models/playlettagmod" "videoweb/pub/models/playletthememod" @@ -16,9 +18,10 @@ import ( ) type idWithTime struct { - Id int64 - Slug string - LastMod time.Time + Id int64 + Slug string + LastMod time.Time + IsOriginal bool } type staticPageEntry struct { @@ -144,7 +147,9 @@ func getURL(sg *SitemapGenerator) { if urls := buildPlayletPathURLs(webAddr, listed.items, "/detail", "weekly", 0.7); len(urls) > 0 { _ = sg.writeSitemapFiles("sitemap-playletdetail", urls) } - if urls := buildPlayletPathURLs(webAddr, listed.items, "/play", "weekly", 0.6); len(urls) > 0 { + // /play 页面需带上默认剧集 ID,否则前端会 301 到 /play/[slug]/[chapterId]/,浪费爬虫抓取预算。 + chapterMap := getPlayletFirstChapterMap(ctx, listed.items) + if urls := buildPlayletPlayURLs(webAddr, listed.items, chapterMap, "weekly", 0.6); len(urls) > 0 { _ = sg.writeSitemapFiles("sitemap-playletplay", urls) } } @@ -194,6 +199,94 @@ func buildPlayletPathURLs(webAddr string, items []idWithTime, prefix string, cha return urls } +// buildPlayletPlayURLs 构建播放页 sitemap URL:有默认剧集时直接拼成 /play/[slug]/[chapterId]/,与前端规范地址一致, +// 避免爬虫经由 301 跳到带剧集 ID 的地址。查不到剧集的短剧跳过,不输出会 301 或 404 的 URL。 +func buildPlayletPlayURLs(webAddr string, items []idWithTime, chapterMap map[int64]int64, changeFreq string, priority float32) []URL { + urls := make([]URL, 0, len(items)) + for _, item := range items { + chapterId, ok := chapterMap[item.Id] + if !ok || chapterId <= 0 { + continue + } + segment := utils.PathSegment(item.Slug, item.Id) + if segment == "" { + continue + } + path := "/play/" + segment + "/" + strconv.FormatInt(chapterId, 10) + "/" + lastMod := formatLastMod(item.LastMod) + if lastMod == "" { + lastMod = time.Now().Format(time.DateOnly) + } + urls = append(urls, URL{ + Loc: utils.JoinUrlPathKeepSlash(webAddr, path), + LastMod: lastMod, + ChangeFreq: changeFreq, + Priority: priority, + }) + } + return urls +} + +// getPlayletFirstChapterMap 批量查询每部短剧的默认(首集)剧集 ID,与前端 /play/[slug]/ 的 301 目标保持一致: +// 按 currentEpisode 升序取首条;原创短剧要求 watermarkVideoUrl 非空(未处理好的集数不露出,避免前端再次跳转)。 +func getPlayletFirstChapterMap(ctx context.Context, items []idWithTime) map[int64]int64 { + result := make(map[int64]int64, len(items)) + if len(items) == 0 { + return result + } + ids := make([]int64, 0, len(items)) + originalSet := make(map[int64]struct{}, len(items)) + for _, item := range items { + if item.Id <= 0 { + continue + } + ids = append(ids, item.Id) + if item.IsOriginal { + originalSet[item.Id] = struct{}{} + } + } + const batchSize = 200 + for start := 0; start < len(ids); start += batchSize { + end := start + batchSize + if end > len(ids) { + end = len(ids) + } + batch := ids[start:end] + list, err := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{ + playletchaptermod.FN.PlayletChapter.PlayletId: bson.M{"$in": batch}, + }, options.Find(). + SetProjection(bson.M{ + playletchaptermod.FN.PlayletChapter.Id: 1, + playletchaptermod.FN.PlayletChapter.PlayletId: 1, + playletchaptermod.FN.PlayletChapter.CurrentEpisode: 1, + playletchaptermod.FN.PlayletChapter.WatermarkVideoUrl: 1, + }). + SetSort(bson.D{ + {Key: playletchaptermod.FN.PlayletChapter.PlayletId, Value: 1}, + {Key: playletchaptermod.FN.PlayletChapter.CurrentEpisode, Value: 1}, + {Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1}, + })) + if err != nil { + continue + } + for _, chapter := range list { + if chapter == nil || chapter.PlayletId <= 0 || chapter.Id <= 0 { + continue + } + if _, ok := result[chapter.PlayletId]; ok { + continue + } + if _, isOriginal := originalSet[chapter.PlayletId]; isOriginal { + if strings.TrimSpace(chapter.WatermarkVideoUrl) == "" { + continue + } + } + result[chapter.PlayletId] = chapter.Id + } + } + return result +} + func formatLastMod(t time.Time) string { if t.IsZero() { return "" @@ -223,6 +316,7 @@ func getPlayletList(ctx context.Context) listedPlaylets { playletmod.FN.Playlet.Slug: 1, playletmod.FN.Playlet.UpdatedAt: 1, playletmod.FN.Playlet.OnlineAt: 1, + playletmod.FN.Playlet.IsOriginal: 1, playletmod.FN.Playlet.ThemeIds: 1, playletmod.FN.Playlet.TagIds: 1, playletmod.FN.Playlet.CategoryIds: 1, @@ -235,7 +329,12 @@ func getPlayletList(ctx context.Context) listedPlaylets { if lastMod.IsZero() { lastMod = info.OnlineAt } - result.items = append(result.items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod}) + result.items = append(result.items, idWithTime{ + Id: info.Id, + Slug: info.Slug, + LastMod: lastMod, + IsOriginal: info.IsOriginal, + }) addListedIDs(result.themeIDs, info.ThemeIds) addListedIDs(result.tagIDs, info.TagIds) addListedIDs(result.categoryIDs, info.CategoryIds)