package sitemap import ( "context" "strings" "sync" "time" "videoweb/pub/models/homeconfigmod" "videoweb/pub/models/playletchaptermod" "videoweb/pub/models/playletmod" "videoweb/pub/models/playlettagmod" "videoweb/pub/models/playletthememod" "videoweb/pub/seo/h5path" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo/options" ) type idWithTime struct { Id int64 Slug string LastMod time.Time IsOriginal bool ChapterID int64 SkipPlaySitemap bool } type staticPageEntry struct { path string changeFreq string priority float32 } func buildStaticURLs(webAddr string, entries []staticPageEntry) []URL { urls := make([]URL, 0, len(entries)) now := sitemapNow() for _, entry := range entries { urls = append(urls, URL{ Loc: h5path.SitemapAbs(webAddr, entry.path), LastMod: now, ChangeFreq: entry.changeFreq, Priority: entry.priority, }) } return urls } func entityPath(prefix, slug string, id int64) string { return h5path.Entity(prefix, slug, id) } // staticPages 对齐 https://sgdj-h5.80sygtm.com/sitemap.xml 的固定页面。 func staticPages() []staticPageEntry { return []staticPageEntry{ {path: "/", changeFreq: "daily", priority: 1}, {path: "/shorts", changeFreq: "daily", priority: 0.8}, {path: "/series", changeFreq: "daily", priority: 0.8}, {path: "/topics", changeFreq: "daily", priority: 0.8}, {path: "/rank", changeFreq: "daily", priority: 0.8}, {path: "/home-road", changeFreq: "daily", priority: 0.8}, {path: "/apps", changeFreq: "daily", priority: 0.8}, {path: "/nav", changeFreq: "monthly", priority: 0.4}, {path: "/info/about-us", changeFreq: "monthly", priority: 0.3}, {path: "/info/user-agreement", changeFreq: "monthly", priority: 0.3}, {path: "/info/privacy-policy", changeFreq: "monthly", priority: 0.3}, {path: "/info/help-center", changeFreq: "monthly", priority: 0.3}, {path: "/info/2257", changeFreq: "monthly", priority: 0.3}, {path: "/info/dmca", changeFreq: "monthly", priority: 0.3}, } } // getURL 拉取上架短剧、专题、标签和首页模块,按线上 H5 路由写入 sitemap。 func getURL(sg *SitemapGenerator) { webAddr := strings.TrimSpace(sg.WebAddr) if webAddr == "" { return } ctx := context.Background() var ( wg sync.WaitGroup listed listedPlaylets themeItems []idWithTime tagItems []idWithTime homeConfigs []homeConfigItem ) wg.Add(4) go func() { defer wg.Done() listed = getPlayletList(ctx) }() go func() { defer wg.Done() themeItems = getThemeList(ctx) }() go func() { defer wg.Done() tagItems = getTagList(ctx) }() go func() { defer wg.Done() homeConfigs = getHomeConfigList(ctx) }() wg.Wait() // 没有上架短剧的题材和标签不写入,避免空列表页被收录。 themeItems = keepWithListedPlaylets(themeItems, listed.themeIDs) tagItems = keepWithListedPlaylets(tagItems, listed.tagIDs) pageURLs := buildStaticURLs(webAddr, staticPages()) for _, item := range themeItems { if path := h5path.Topic(item.Slug, item.Id); path != "" { pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "daily", 0.8)) } } for _, item := range tagItems { if path := h5path.TopicTag(item.Slug, item.Id); path != "" { pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "weekly", 0.6)) } } for _, item := range homeConfigs { if item.Type != homeconfigmod.TypePlaylet { continue } if path := h5path.ShortsMore(item.Slug, item.Id); path != "" { pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "daily", 0.7)) } } if len(pageURLs) > 0 { _ = sg.writeSitemapFile("sitemap-pages.xml", pageURLs) } // 详情页是 /drama/{slug}。各集 /play/{slug}/{episode} 不进 sitemap,避免和详情页重复。 // 原创且第一集无水印的剧不写入,避免空壳页被提交。 populateFirstChapters(ctx, listed.items) if urls := buildDramaURLs(webAddr, listed.items, "weekly", 0.7); len(urls) > 0 { _ = sg.writeSitemapFiles("sitemap-drama", urls) } sg.removeObsoleteSitemaps("sitemap-playletdetail", "sitemap-seriesplay") } type listedPlaylets struct { items []idWithTime themeIDs map[int64]struct{} tagIDs map[int64]struct{} } type homeConfigItem struct { Id int64 Slug string Type int32 LastMod time.Time } func newURL(webAddr, path string, lastMod time.Time, changeFreq string, priority float32) URL { mod := sitemapLastMod(lastMod) if mod == "" { mod = sitemapNow() } return URL{ Loc: h5path.SitemapAbs(webAddr, path), LastMod: mod, ChangeFreq: changeFreq, Priority: priority, } } func keepWithListedPlaylets(items []idWithTime, listed map[int64]struct{}) []idWithTime { if len(items) == 0 || len(listed) == 0 { return nil } out := make([]idWithTime, 0, len(items)) for _, item := range items { if _, ok := listed[item.Id]; ok { out = append(out, item) } } return out } func addListedIDs(dst map[int64]struct{}, ids []int64) { for _, id := range ids { if id > 0 { dst[id] = struct{}{} } } } func buildDramaURLs(webAddr string, items []idWithTime, changeFreq string, priority float32) []URL { urls := make([]URL, 0, len(items)) for _, item := range items { if item.SkipPlaySitemap { continue } path := h5path.Drama(item.Slug, item.Id) if path == "" { continue } urls = append(urls, newURL(webAddr, path, item.LastMod, changeFreq, priority)) } return urls } func skipOriginalChapterWithoutWatermark(isOriginal bool, watermarkURL string) bool { return isOriginal && strings.TrimSpace(watermarkURL) == "" } // populateFirstChapters 每剧只看第一集;原创且无水印则不写详情 URL。 func populateFirstChapters(ctx context.Context, items []idWithTime) { for start := 0; start < len(items); start += 500 { end := start + 500 if end > len(items) { end = len(items) } ids := make([]int64, 0, end-start) byID := make(map[int64]*idWithTime, end-start) for i := start; i < end; i++ { if items[i].Id <= 0 { continue } ids = append(ids, items[i].Id) byID[items[i].Id] = &items[i] } if len(ids) == 0 { continue } chapters, err := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{ playletchaptermod.FN.PlayletChapter.PlayletId: bson.M{"$in": ids}, }, options.Find(). SetSort(bson.D{ {Key: playletchaptermod.FN.PlayletChapter.CurrentEpisode, Value: 1}, {Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1}, }). SetProjection(bson.M{ playletchaptermod.FN.PlayletChapter.Id: 1, playletchaptermod.FN.PlayletChapter.PlayletId: 1, playletchaptermod.FN.PlayletChapter.WatermarkVideoUrl: 1, })) if err != nil { return } seenFirst := make(map[int64]struct{}, len(ids)) for _, ch := range chapters { if ch == nil || ch.Id <= 0 { continue } item := byID[ch.PlayletId] if item == nil { continue } if _, ok := seenFirst[ch.PlayletId]; ok { continue } seenFirst[ch.PlayletId] = struct{}{} if skipOriginalChapterWithoutWatermark(item.IsOriginal, ch.WatermarkVideoUrl) { item.SkipPlaySitemap = true continue } item.ChapterID = ch.Id } } } func getPlayletList(ctx context.Context) listedPlaylets { result := listedPlaylets{ items: make([]idWithTime, 0), themeIDs: make(map[int64]struct{}), tagIDs: make(map[int64]struct{}), } filter := bson.M{ playletmod.FN.Playlet.Status: playletmod.StatusUp, } pageSize := int64(1000) for i := int64(0); i < 500; i++ { skip := i * pageSize list, _ := playletmod.FindManyPlaylet(ctx, filter, options.Find(). SetSort(bson.M{playletmod.FN.Playlet.Id: -1}). SetSkip(skip). SetLimit(pageSize). SetProjection(bson.M{ playletmod.FN.Playlet.Id: 1, playletmod.FN.Playlet.Slug: 1, playletmod.FN.Playlet.IsOriginal: 1, playletmod.FN.Playlet.UpdatedAt: 1, playletmod.FN.Playlet.OnlineAt: 1, playletmod.FN.Playlet.ThemeIds: 1, playletmod.FN.Playlet.TagIds: 1, })) for _, info := range list { if info == nil || info.Id <= 0 { continue } lastMod := info.UpdatedAt if lastMod.IsZero() { lastMod = info.OnlineAt } result.items = append(result.items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod, IsOriginal: info.IsOriginal}) addListedIDs(result.themeIDs, info.ThemeIds) addListedIDs(result.tagIDs, info.TagIds) } if len(list) < int(pageSize) { break } } return result } func getThemeList(ctx context.Context) []idWithTime { filter := bson.M{ playletthememod.FN.PlayletTheme.Status: true, } pageSize := int64(1000) items := make([]idWithTime, 0) for i := int64(0); i < 100; i++ { skip := i * pageSize list, _ := playletthememod.FindManyPlayletTheme(ctx, filter, options.Find(). SetSort(bson.M{playletthememod.FN.PlayletTheme.Rank: -1}). SetSkip(skip). SetLimit(pageSize)) for _, info := range list { if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" { continue } items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}) } if len(list) < int(pageSize) { break } } return items } func getTagList(ctx context.Context) []idWithTime { filter := bson.M{ playlettagmod.FN.PlayletTag.Status: true, } pageSize := int64(1000) items := make([]idWithTime, 0) for i := int64(0); i < 100; i++ { skip := i * pageSize list, _ := playlettagmod.FindManyPlayletTag(ctx, filter, options.Find(). SetSort(bson.M{playlettagmod.FN.PlayletTag.Rank: -1}). SetSkip(skip). SetLimit(pageSize)) for _, info := range list { if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" { continue } items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}) } if len(list) < int(pageSize) { break } } return items } func getHomeConfigList(ctx context.Context) []homeConfigItem { list, _ := homeconfigmod.FindManyHomeConfig(ctx, bson.M{ homeconfigmod.FN.HomeConfig.Status: true, homeconfigmod.FN.HomeConfig.Type: bson.M{ "$in": []int32{homeconfigmod.TypePlaylet}, }, }, options.Find().SetSort(bson.M{homeconfigmod.FN.HomeConfig.Rank: -1})) items := make([]homeConfigItem, 0, len(list)) for _, info := range list { if info == nil || info.Id <= 0 { continue } items = append(items, homeConfigItem{Id: info.Id, Slug: info.Slug, Type: info.Type, LastMod: info.UpdatedAt}) } return items }