package sitemap import ( "context" "strconv" "strings" "sync" "time" "videoweb/pub/common/utils" "videoweb/pub/models/playletcategorymod" "videoweb/pub/models/playletchaptermod" "videoweb/pub/models/playletmod" "videoweb/pub/models/playlettagmod" "videoweb/pub/models/playletthememod" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo/options" ) type idWithTime struct { Id int64 Slug string LastMod time.Time IsOriginal bool } type staticPageEntry struct { path string changeFreq string priority float32 } func buildStaticURLs(webAddr string, entries []staticPageEntry) []URL { urls := make([]URL, 0, len(entries)) now := time.Now().Format(time.DateOnly) for _, entry := range entries { urls = append(urls, URL{ Loc: utils.JoinUrlPathKeepSlash(webAddr, entry.path), LastMod: now, ChangeFreq: entry.changeFreq, Priority: entry.priority, }) } return urls } func entityPath(prefix, slug string, id int64) string { if segment := utils.PathSegment(slug, id); segment != "" { return strings.TrimSuffix(prefix, "/") + "/" + segment + "/" } if strings.HasSuffix(prefix, "/") { return prefix } return prefix + "/" } func hgdjStaticPages() []staticPageEntry { return []staticPageEntry{ {path: "/", changeFreq: "daily", priority: 1.0}, {path: "/list/ai-short-play/", changeFreq: "daily", priority: 0.9}, {path: "/list/ai-short-play/popular/", changeFreq: "daily", priority: 0.8}, {path: "/list/ai-short-play/original/", changeFreq: "daily", priority: 0.8}, {path: "/series/", changeFreq: "daily", priority: 0.9}, {path: "/series/popular/", changeFreq: "daily", priority: 0.8}, {path: "/series/original/", changeFreq: "daily", priority: 0.8}, {path: "/rank/", changeFreq: "daily", priority: 0.8}, {path: "/today-update/", changeFreq: "daily", priority: 0.8}, {path: "/topic/", changeFreq: "daily", priority: 0.8}, {path: "/application/", changeFreq: "weekly", priority: 0.6}, {path: "/home-road/", changeFreq: "weekly", priority: 0.6}, {path: "/download-app/", changeFreq: "monthly", priority: 0.5}, {path: "/nav/", changeFreq: "monthly", priority: 0.5}, {path: "/about-us/", changeFreq: "yearly", priority: 0.3}, {path: "/help-center/", changeFreq: "yearly", priority: 0.3}, } } // getURL 并发拉取短剧数据并写入 sitemap 分片,URL 规则对齐 jha-1043-hgdj-web。 func getURL(sg *SitemapGenerator) { webAddr := strings.TrimSpace(sg.WebAddr) if webAddr == "" { return } ctx := context.Background() var ( wg sync.WaitGroup listed listedPlaylets themeItems []idWithTime tagItems []idWithTime categoryItems []idWithTime ) wg.Add(4) go func() { defer wg.Done() listed = getPlayletList(ctx) }() go func() { defer wg.Done() themeItems = getThemeList(ctx) }() go func() { defer wg.Done() tagItems = getTagList(ctx) }() go func() { defer wg.Done() categoryItems = getCategoryList(ctx) }() wg.Wait() // 专题/标签没有上架短剧时不写入,避免空列表页被收录。 // 分类页无绑定短剧时会回退展示全部上架短剧,因此启用分类都写入。 themeItems = keepWithListedPlaylets(themeItems, listed.themeIDs) tagItems = keepWithListedPlaylets(tagItems, listed.tagIDs) pageURLs := buildStaticURLs(webAddr, hgdjStaticPages()) for _, item := range themeItems { pageURLs = append(pageURLs, URL{ Loc: utils.JoinUrlPathKeepSlash(webAddr, entityPath("/topic", item.Slug, item.Id)), LastMod: formatLastMod(item.LastMod), ChangeFreq: "daily", Priority: 0.8, }) } for _, item := range categoryItems { pageURLs = append(pageURLs, URL{ Loc: utils.JoinUrlPathKeepSlash(webAddr, entityPath("/category", item.Slug, item.Id)), LastMod: formatLastMod(item.LastMod), ChangeFreq: "daily", Priority: 0.8, }) } for _, item := range tagItems { pageURLs = append(pageURLs, URL{ Loc: utils.JoinUrlPathKeepSlash(webAddr, entityPath("/tag", item.Slug, item.Id)), LastMod: formatLastMod(item.LastMod), ChangeFreq: "weekly", Priority: 0.6, }) } if len(pageURLs) > 0 { _ = sg.writeSitemapFile("sitemap-pages.xml", pageURLs) } if urls := buildPlayletPathURLs(webAddr, listed.items, "/detail", "weekly", 0.7); len(urls) > 0 { _ = sg.writeSitemapFiles("sitemap-playletdetail", urls) } // /play 页面需带上默认剧集 ID,否则前端会 301 到 /play/[slug]/[chapterId]/,浪费爬虫抓取预算。 chapterMap := getPlayletFirstChapterMap(ctx, listed.items) if urls := buildPlayletPlayURLs(webAddr, listed.items, chapterMap, "weekly", 0.6); len(urls) > 0 { _ = sg.writeSitemapFiles("sitemap-playletplay", urls) } } type listedPlaylets struct { items []idWithTime themeIDs map[int64]struct{} tagIDs map[int64]struct{} categoryIDs map[int64]struct{} } func keepWithListedPlaylets(items []idWithTime, listed map[int64]struct{}) []idWithTime { if len(items) == 0 || len(listed) == 0 { return nil } out := make([]idWithTime, 0, len(items)) for _, item := range items { if _, ok := listed[item.Id]; ok { out = append(out, item) } } return out } func addListedIDs(dst map[int64]struct{}, ids []int64) { for _, id := range ids { if id > 0 { dst[id] = struct{}{} } } } func buildPlayletPathURLs(webAddr string, items []idWithTime, prefix string, changeFreq string, priority float32) []URL { urls := make([]URL, 0, len(items)) for _, item := range items { lastMod := formatLastMod(item.LastMod) if lastMod == "" { lastMod = time.Now().Format(time.DateOnly) } urls = append(urls, URL{ Loc: utils.JoinUrlPathKeepSlash(webAddr, entityPath(prefix, item.Slug, item.Id)), LastMod: lastMod, ChangeFreq: changeFreq, Priority: priority, }) } return urls } // buildPlayletPlayURLs 构建播放页 sitemap URL:有默认剧集时直接拼成 /play/[slug]/[chapterId]/,与前端规范地址一致, // 避免爬虫经由 301 跳到带剧集 ID 的地址。查不到剧集的短剧跳过,不输出会 301 或 404 的 URL。 func buildPlayletPlayURLs(webAddr string, items []idWithTime, chapterMap map[int64]int64, changeFreq string, priority float32) []URL { urls := make([]URL, 0, len(items)) for _, item := range items { chapterId, ok := chapterMap[item.Id] if !ok || chapterId <= 0 { continue } segment := utils.PathSegment(item.Slug, item.Id) if segment == "" { continue } path := "/play/" + segment + "/" + strconv.FormatInt(chapterId, 10) + "/" lastMod := formatLastMod(item.LastMod) if lastMod == "" { lastMod = time.Now().Format(time.DateOnly) } urls = append(urls, URL{ Loc: utils.JoinUrlPathKeepSlash(webAddr, path), LastMod: lastMod, ChangeFreq: changeFreq, Priority: priority, }) } return urls } // getPlayletFirstChapterMap 批量查询每部短剧的默认(首集)剧集 ID,与前端 /play/[slug]/ 的 301 目标保持一致: // 按 currentEpisode 升序取首条;原创短剧要求 watermarkVideoUrl 非空(未处理好的集数不露出,避免前端再次跳转)。 func getPlayletFirstChapterMap(ctx context.Context, items []idWithTime) map[int64]int64 { result := make(map[int64]int64, len(items)) if len(items) == 0 { return result } ids := make([]int64, 0, len(items)) originalSet := make(map[int64]struct{}, len(items)) for _, item := range items { if item.Id <= 0 { continue } ids = append(ids, item.Id) if item.IsOriginal { originalSet[item.Id] = struct{}{} } } const batchSize = 200 for start := 0; start < len(ids); start += batchSize { end := start + batchSize if end > len(ids) { end = len(ids) } batch := ids[start:end] list, err := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{ playletchaptermod.FN.PlayletChapter.PlayletId: bson.M{"$in": batch}, }, options.Find(). SetProjection(bson.M{ playletchaptermod.FN.PlayletChapter.Id: 1, playletchaptermod.FN.PlayletChapter.PlayletId: 1, playletchaptermod.FN.PlayletChapter.CurrentEpisode: 1, playletchaptermod.FN.PlayletChapter.WatermarkVideoUrl: 1, }). SetSort(bson.D{ {Key: playletchaptermod.FN.PlayletChapter.PlayletId, Value: 1}, {Key: playletchaptermod.FN.PlayletChapter.CurrentEpisode, Value: 1}, {Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1}, })) if err != nil { continue } for _, chapter := range list { if chapter == nil || chapter.PlayletId <= 0 || chapter.Id <= 0 { continue } if _, ok := result[chapter.PlayletId]; ok { continue } if _, isOriginal := originalSet[chapter.PlayletId]; isOriginal { if strings.TrimSpace(chapter.WatermarkVideoUrl) == "" { continue } } result[chapter.PlayletId] = chapter.Id } } return result } func formatLastMod(t time.Time) string { if t.IsZero() { return "" } return t.Format(time.DateOnly) } func getPlayletList(ctx context.Context) listedPlaylets { result := listedPlaylets{ items: make([]idWithTime, 0), themeIDs: make(map[int64]struct{}), tagIDs: make(map[int64]struct{}), categoryIDs: make(map[int64]struct{}), } filter := bson.M{ playletmod.FN.Playlet.Status: playletmod.StatusUp, } pageSize := int64(1000) for i := int64(0); i < 500; i++ { skip := i * pageSize list, _ := playletmod.FindManyPlaylet(ctx, filter, options.Find(). SetSort(bson.M{playletmod.FN.Playlet.Id: -1}). SetSkip(skip). SetLimit(pageSize). SetProjection(bson.M{ playletmod.FN.Playlet.Id: 1, playletmod.FN.Playlet.Slug: 1, playletmod.FN.Playlet.UpdatedAt: 1, playletmod.FN.Playlet.OnlineAt: 1, playletmod.FN.Playlet.IsOriginal: 1, playletmod.FN.Playlet.ThemeIds: 1, playletmod.FN.Playlet.TagIds: 1, playletmod.FN.Playlet.CategoryIds: 1, })) for _, info := range list { if info == nil || info.Id <= 0 { continue } lastMod := info.UpdatedAt if lastMod.IsZero() { lastMod = info.OnlineAt } result.items = append(result.items, idWithTime{ Id: info.Id, Slug: info.Slug, LastMod: lastMod, IsOriginal: info.IsOriginal, }) addListedIDs(result.themeIDs, info.ThemeIds) addListedIDs(result.tagIDs, info.TagIds) addListedIDs(result.categoryIDs, info.CategoryIds) } if len(list) < int(pageSize) { break } } return result } func getThemeList(ctx context.Context) []idWithTime { filter := bson.M{ playletthememod.FN.PlayletTheme.Status: true, } pageSize := int64(1000) items := make([]idWithTime, 0) for i := int64(0); i < 100; i++ { skip := i * pageSize list, _ := playletthememod.FindManyPlayletTheme(ctx, filter, options.Find(). SetSort(bson.M{playletthememod.FN.PlayletTheme.Rank: -1}). SetSkip(skip). SetLimit(pageSize)) for _, info := range list { if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" { continue } items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}) } if len(list) < int(pageSize) { break } } return items } func getTagList(ctx context.Context) []idWithTime { filter := bson.M{ playlettagmod.FN.PlayletTag.Status: true, } pageSize := int64(1000) items := make([]idWithTime, 0) for i := int64(0); i < 100; i++ { skip := i * pageSize list, _ := playlettagmod.FindManyPlayletTag(ctx, filter, options.Find(). SetSort(bson.M{playlettagmod.FN.PlayletTag.Rank: -1}). SetSkip(skip). SetLimit(pageSize)) for _, info := range list { if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" { continue } items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}) } if len(list) < int(pageSize) { break } } return items } func getCategoryList(ctx context.Context) []idWithTime { filter := bson.M{ playletcategorymod.FN.PlayletCategory.Status: true, } pageSize := int64(1000) items := make([]idWithTime, 0) for i := int64(0); i < 100; i++ { skip := i * pageSize list, _ := playletcategorymod.FindManyPlayletCategory(ctx, filter, options.Find(). SetSort(bson.M{playletcategorymod.FN.PlayletCategory.Rank: -1}). SetSkip(skip). SetLimit(pageSize)) for _, info := range list { if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" { continue } items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}) } if len(list) < int(pageSize) { break } } return items }