package sitemap import ( "context" "strings" "sync" "time" "videoweb/pub/models/homeconfigmod" "videoweb/pub/models/playletchaptermod" "videoweb/pub/models/playletmod" "videoweb/pub/models/playlettagmod" "videoweb/pub/models/playletthememod" "videoweb/pub/seo/h5path" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo/options" ) type playEpisode struct { Episode int64 LastMod time.Time } type idWithTime struct { Id int64 Slug string LastMod time.Time IsOriginal bool Episodes []playEpisode } type staticPageEntry struct { path string changeFreq string priority float32 } func buildStaticURLs(webAddr string, entries []staticPageEntry) []URL { urls := make([]URL, 0, len(entries)) now := time.Now().Format(time.DateOnly) for _, entry := range entries { urls = append(urls, URL{ Loc: h5path.SitemapAbs(webAddr, entry.path), LastMod: now, ChangeFreq: entry.changeFreq, Priority: entry.priority, }) } return urls } func entityPath(prefix, slug string, id int64) string { return h5path.Entity(prefix, slug, id) } func siteStaticPages() []staticPageEntry { return []staticPageEntry{ {path: "/", changeFreq: "daily", priority: 1.0}, {path: "/shorts", changeFreq: "daily", priority: 0.9}, {path: "/shorts/popular", changeFreq: "daily", priority: 0.7}, {path: "/shorts/original", changeFreq: "daily", priority: 0.7}, {path: "/shorts/random", changeFreq: "daily", priority: 0.7}, {path: "/series", changeFreq: "daily", priority: 0.9}, {path: "/series/popular", changeFreq: "daily", priority: 0.7}, {path: "/series/original", changeFreq: "daily", priority: 0.7}, {path: "/series/random", changeFreq: "daily", priority: 0.7}, {path: "/topics", changeFreq: "daily", priority: 0.8}, {path: "/rank", changeFreq: "daily", priority: 0.8}, {path: "/rank/new", changeFreq: "daily", priority: 0.7}, {path: "/rank/binge", changeFreq: "daily", priority: 0.7}, {path: "/home-road", changeFreq: "weekly", priority: 0.6}, {path: "/apps", changeFreq: "weekly", priority: 0.6}, {path: "/nav", changeFreq: "weekly", priority: 0.5}, {path: "/info/about-us", changeFreq: "yearly", priority: 0.3}, {path: "/info/user-agreement", changeFreq: "yearly", priority: 0.3}, {path: "/info/privacy-policy", changeFreq: "yearly", priority: 0.3}, {path: "/info/help-center", changeFreq: "yearly", priority: 0.3}, {path: "/info/2257", changeFreq: "yearly", priority: 0.3}, {path: "/info/dmca", changeFreq: "yearly", priority: 0.3}, } } // getURL 并发拉取短剧数据并写入 sitemap 分片。 // 路径对齐 https://91dj-h5.80sygtm.com/ 上有独立 canonical、且未被 noindex 的页面。 func getURL(sg *SitemapGenerator) { webAddr := strings.TrimSpace(sg.WebAddr) if webAddr == "" { return } ctx := context.Background() var ( wg sync.WaitGroup listed listedPlaylets themeItems []idWithTime tagItems []idWithTime homeConfigs []homeConfigItem ) wg.Add(4) go func() { defer wg.Done() listed = getPlayletList(ctx) }() go func() { defer wg.Done() themeItems = getThemeList(ctx) }() go func() { defer wg.Done() tagItems = getTagList(ctx) }() go func() { defer wg.Done() homeConfigs = getHomeConfigList(ctx) }() wg.Wait() // 题材/标签没有上架短剧时不写入,避免空列表页被收录。 themeItems = keepWithListedPlaylets(themeItems, listed.themeIDs) tagItems = keepWithListedPlaylets(tagItems, listed.tagIDs) pageURLs := buildStaticURLs(webAddr, siteStaticPages()) for _, item := range themeItems { pageURLs = append(pageURLs, newURL(webAddr, h5path.TopicDetail(item.Slug, item.Id), item.LastMod, "daily", 0.8)) } for _, item := range tagItems { if path := h5path.Tag(item.Slug, item.Id); path != "" { pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "weekly", 0.6)) } } for _, item := range homeConfigs { path := "" switch item.Type { case homeconfigmod.TypePlaylet: path = h5path.MoreList(item.Slug, item.Id) } if path != "" { pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "daily", 0.7)) } } if len(pageURLs) > 0 { _ = sg.writeSitemapFile("sitemap-pages.xml", pageURLs) } // 详情页 /short-drama/{slug} 与播放页 /short-drama/{slug}/play/{集数} 都有独立 canonical。 // 原创且该集没有水印成片时不提交播放页,避免空壳播放地址被收录。 populatePlayEpisodes(ctx, listed.items) if urls := buildPlayletPathURLs(webAddr, listed.items, "weekly", 0.8); len(urls) > 0 { _ = sg.writeSitemapFiles("sitemap-playletdetail", urls) } if urls := buildPlayletEpisodeURLs(webAddr, listed.items, "weekly", 0.6); len(urls) > 0 { _ = sg.writeSitemapFiles("sitemap-playletplay", urls) } _ = sg.writeSitemapFiles("sitemap-seriesplay", nil) } type listedPlaylets struct { items []idWithTime themeIDs map[int64]struct{} tagIDs map[int64]struct{} } type homeConfigItem struct { Id int64 Slug string Type int32 LastMod time.Time } func newURL(webAddr, path string, lastMod time.Time, changeFreq string, priority float32) URL { mod := formatLastMod(lastMod) if mod == "" { mod = time.Now().Format(time.DateOnly) } return URL{ Loc: h5path.SitemapAbs(webAddr, path), LastMod: mod, ChangeFreq: changeFreq, Priority: priority, } } func keepWithListedPlaylets(items []idWithTime, listed map[int64]struct{}) []idWithTime { if len(items) == 0 || len(listed) == 0 { return nil } out := make([]idWithTime, 0, len(items)) for _, item := range items { if _, ok := listed[item.Id]; ok { out = append(out, item) } } return out } func addListedIDs(dst map[int64]struct{}, ids []int64) { for _, id := range ids { if id > 0 { dst[id] = struct{}{} } } } func buildPlayletPathURLs(webAddr string, items []idWithTime, changeFreq string, priority float32) []URL { urls := make([]URL, 0, len(items)) for _, item := range items { path := h5path.Playlet(item.Slug, item.Id) if path == "" { continue } urls = append(urls, newURL(webAddr, path, item.LastMod, changeFreq, priority)) } return urls } func buildPlayletEpisodeURLs(webAddr string, items []idWithTime, changeFreq string, priority float32) []URL { urls := make([]URL, 0) for _, item := range items { for _, episode := range item.Episodes { path := h5path.PlayletPlay(item.Slug, item.Id, episode.Episode) if path == "" { continue } lastMod := episode.LastMod if lastMod.IsZero() { lastMod = item.LastMod } urls = append(urls, newURL(webAddr, path, lastMod, changeFreq, priority)) } } return urls } func skipOriginalChapterWithoutWatermark(isOriginal bool, watermarkURL string) bool { return isOriginal && strings.TrimSpace(watermarkURL) == "" } // populatePlayEpisodes 收集上架剧集的播放页。原创且没有水印成片的集不提交。 func populatePlayEpisodes(ctx context.Context, items []idWithTime) { for start := 0; start < len(items); start += 500 { end := start + 500 if end > len(items) { end = len(items) } ids := make([]int64, 0, end-start) byID := make(map[int64]*idWithTime, end-start) for i := start; i < end; i++ { if items[i].Id <= 0 { continue } ids = append(ids, items[i].Id) byID[items[i].Id] = &items[i] } if len(ids) == 0 { continue } filter := bson.M{ playletchaptermod.FN.PlayletChapter.PlayletId: bson.M{"$in": ids}, } for key, value := range playletchaptermod.VisibleStatusFilter() { filter[key] = value } chapters, err := playletchaptermod.FindManyPlayletChapter(ctx, filter, options.Find(). SetSort(bson.D{ {Key: playletchaptermod.FN.PlayletChapter.CurrentEpisode, Value: 1}, {Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1}, }). SetProjection(bson.M{ playletchaptermod.FN.PlayletChapter.Id: 1, playletchaptermod.FN.PlayletChapter.PlayletId: 1, playletchaptermod.FN.PlayletChapter.CurrentEpisode: 1, playletchaptermod.FN.PlayletChapter.WatermarkVideoUrl: 1, playletchaptermod.FN.PlayletChapter.UpdatedAt: 1, })) if err != nil { return } seen := make(map[int64]map[int64]struct{}, len(ids)) for _, ch := range chapters { if ch == nil || ch.PlayletId <= 0 || ch.CurrentEpisode <= 0 { continue } item := byID[ch.PlayletId] if item == nil || skipOriginalChapterWithoutWatermark(item.IsOriginal, ch.WatermarkVideoUrl) { continue } if seen[ch.PlayletId] == nil { seen[ch.PlayletId] = make(map[int64]struct{}) } if _, ok := seen[ch.PlayletId][ch.CurrentEpisode]; ok { continue } seen[ch.PlayletId][ch.CurrentEpisode] = struct{}{} item.Episodes = append(item.Episodes, playEpisode{Episode: ch.CurrentEpisode, LastMod: ch.UpdatedAt}) } } } func formatLastMod(t time.Time) string { if t.IsZero() { return "" } return t.Format(time.DateOnly) } func getPlayletList(ctx context.Context) listedPlaylets { result := listedPlaylets{ items: make([]idWithTime, 0), themeIDs: make(map[int64]struct{}), tagIDs: make(map[int64]struct{}), } filter := bson.M{ playletmod.FN.Playlet.Status: playletmod.StatusUp, } pageSize := int64(1000) for i := int64(0); i < 500; i++ { skip := i * pageSize list, _ := playletmod.FindManyPlaylet(ctx, filter, options.Find(). SetSort(bson.M{playletmod.FN.Playlet.Id: -1}). SetSkip(skip). SetLimit(pageSize). SetProjection(bson.M{ playletmod.FN.Playlet.Id: 1, playletmod.FN.Playlet.Slug: 1, playletmod.FN.Playlet.IsOriginal: 1, playletmod.FN.Playlet.UpdatedAt: 1, playletmod.FN.Playlet.OnlineAt: 1, playletmod.FN.Playlet.ThemeIds: 1, playletmod.FN.Playlet.TagIds: 1, })) for _, info := range list { if info == nil || info.Id <= 0 { continue } lastMod := info.UpdatedAt if lastMod.IsZero() { lastMod = info.OnlineAt } result.items = append(result.items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod, IsOriginal: info.IsOriginal}) addListedIDs(result.themeIDs, info.ThemeIds) addListedIDs(result.tagIDs, info.TagIds) } if len(list) < int(pageSize) { break } } return result } func getThemeList(ctx context.Context) []idWithTime { filter := bson.M{ playletthememod.FN.PlayletTheme.Status: true, } pageSize := int64(1000) items := make([]idWithTime, 0) for i := int64(0); i < 100; i++ { skip := i * pageSize list, _ := playletthememod.FindManyPlayletTheme(ctx, filter, options.Find(). SetSort(bson.M{playletthememod.FN.PlayletTheme.Rank: -1}). SetSkip(skip). SetLimit(pageSize)) for _, info := range list { if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" { continue } items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}) } if len(list) < int(pageSize) { break } } return items } func getTagList(ctx context.Context) []idWithTime { filter := bson.M{ playlettagmod.FN.PlayletTag.Status: true, } pageSize := int64(1000) items := make([]idWithTime, 0) for i := int64(0); i < 100; i++ { skip := i * pageSize list, _ := playlettagmod.FindManyPlayletTag(ctx, filter, options.Find(). SetSort(bson.M{playlettagmod.FN.PlayletTag.Rank: -1}). SetSkip(skip). SetLimit(pageSize)) for _, info := range list { if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" { continue } items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}) } if len(list) < int(pageSize) { break } } return items } func getHomeConfigList(ctx context.Context) []homeConfigItem { list, _ := homeconfigmod.FindManyHomeConfig(ctx, bson.M{ homeconfigmod.FN.HomeConfig.Status: true, homeconfigmod.FN.HomeConfig.Type: bson.M{ "$in": []int32{homeconfigmod.TypePlaylet}, }, }, options.Find().SetSort(bson.M{homeconfigmod.FN.HomeConfig.Rank: -1})) items := make([]homeConfigItem, 0, len(list)) for _, info := range list { if info == nil || info.Id <= 0 { continue } items = append(items, homeConfigItem{Id: info.Id, Slug: info.Slug, Type: info.Type, LastMod: info.UpdatedAt}) } return items }