package sitemap import ( "context" "strings" "sync" "time" "videoweb/pub/models/playletchaptermod" "videoweb/pub/models/playletmod" "videoweb/pub/models/playlettagmod" "videoweb/pub/models/playletthememod" "videoweb/pub/seo/h5path" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo/options" ) type idWithTime struct { Id int64 Slug string LastMod time.Time IsOriginal bool ChapterID int64 SkipPlaySitemap bool } type staticPageEntry struct { path string changeFreq string priority float32 } func buildStaticURLs(webAddr string, entries []staticPageEntry) []URL { urls := make([]URL, 0, len(entries)) now := time.Now().Format(time.DateOnly) for _, entry := range entries { urls = append(urls, URL{ Loc: h5path.SitemapAbs(webAddr, entry.path), LastMod: now, ChangeFreq: entry.changeFreq, Priority: entry.priority, }) } return urls } func entityPath(prefix, slug string, id int64) string { return h5path.Entity(prefix, slug, id) } // jgdjStaticPages 固定页面,对齐 jgdj1.net 上可收录的地址。 // /topics 专题列表、/config 首页配置、充值和反馈为 noindex;/rank 会规范到 /rank/hot。 func jgdjStaticPages() []staticPageEntry { return []staticPageEntry{ {path: "/", changeFreq: "daily", priority: 1.0}, {path: "/shorts", changeFreq: "daily", priority: 0.9}, {path: "/rank/hot", changeFreq: "daily", priority: 0.8}, {path: "/rank/new", changeFreq: "daily", priority: 0.7}, {path: "/rank/follow", changeFreq: "daily", priority: 0.7}, {path: "/info/about-us", changeFreq: "yearly", priority: 0.3}, {path: "/info/user-agreement", changeFreq: "yearly", priority: 0.3}, {path: "/info/privacy-policy", changeFreq: "yearly", priority: 0.3}, {path: "/info/help-center", changeFreq: "yearly", priority: 0.3}, {path: "/info/2257", changeFreq: "yearly", priority: 0.3}, {path: "/info/dmca", changeFreq: "yearly", priority: 0.3}, } } func contentPath(prefix, slug string, id int64) string { if h5path.Segment(slug, id) == "" { return "" } return h5path.Entity(prefix, slug, id) } // getURL 并发拉取短剧数据并写入 sitemap 分片。 // 分片对齐 jgdj1.net:sitemap-pages、sitemap-playletdetail。 func getURL(sg *SitemapGenerator) { webAddr := strings.TrimSpace(sg.WebAddr) if webAddr == "" { return } ctx := context.Background() var ( wg sync.WaitGroup listed listedPlaylets themeItems []idWithTime tagItems []idWithTime ) wg.Add(3) go func() { defer wg.Done() listed = getPlayletList(ctx) }() go func() { defer wg.Done() themeItems = getThemeList(ctx) }() go func() { defer wg.Done() tagItems = getTagList(ctx) }() wg.Wait() // 题材/标签没有上架短剧时不写入,避免空列表页被收录。 themeItems = keepWithListedPlaylets(themeItems, listed.themeIDs) tagItems = keepWithListedPlaylets(tagItems, listed.tagIDs) pageURLs := buildStaticURLs(webAddr, jgdjStaticPages()) for _, item := range themeItems { if path := contentPath("/topics", item.Slug, item.Id); path != "" { pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "daily", 0.8)) } } for _, item := range tagItems { if path := contentPath("/tag", item.Slug, item.Id); path != "" { pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "weekly", 0.6)) } } if len(pageURLs) > 0 { _ = sg.writeSitemapFile("sitemap-pages.xml", pageURLs) } // 分集页不进 sitemap,只保留 /detail/{slug}。 // 原创且第一集无水印的详情不进 sitemap,避免空壳页被提交。 populateFirstChapters(ctx, listed.items) if urls := buildPlayletPathURLs(webAddr, listed.items, "weekly", 0.7); len(urls) > 0 { _ = sg.writeSitemapFiles("sitemap-playletdetail", urls) } } type listedPlaylets struct { items []idWithTime themeIDs map[int64]struct{} tagIDs map[int64]struct{} } func newURL(webAddr, path string, lastMod time.Time, changeFreq string, priority float32) URL { mod := formatLastMod(lastMod) if mod == "" { mod = time.Now().Format(time.DateOnly) } return URL{ Loc: h5path.SitemapAbs(webAddr, path), LastMod: mod, ChangeFreq: changeFreq, Priority: priority, } } func keepWithListedPlaylets(items []idWithTime, listed map[int64]struct{}) []idWithTime { if len(items) == 0 || len(listed) == 0 { return nil } out := make([]idWithTime, 0, len(items)) for _, item := range items { if _, ok := listed[item.Id]; ok { out = append(out, item) } } return out } func addListedIDs(dst map[int64]struct{}, ids []int64) { for _, id := range ids { if id > 0 { dst[id] = struct{}{} } } } func buildPlayletPathURLs(webAddr string, items []idWithTime, changeFreq string, priority float32) []URL { urls := make([]URL, 0, len(items)) for _, item := range items { if item.SkipPlaySitemap { continue } path := contentPath("/detail", item.Slug, item.Id) if path == "" { continue } urls = append(urls, newURL(webAddr, path, item.LastMod, changeFreq, priority)) } return urls } func playletIDs(items []idWithTime) []int64 { ids := make([]int64, 0, len(items)) for _, item := range items { if item.Id > 0 { ids = append(ids, item.Id) } } return ids } func skipOriginalChapterWithoutWatermark(isOriginal bool, watermarkURL string) bool { return isOriginal && strings.TrimSpace(watermarkURL) == "" } // populateFirstChapters 每剧只看第一集;原创且无水印则不写播放 URL。 func populateFirstChapters(ctx context.Context, items []idWithTime) { for start := 0; start < len(items); start += 500 { end := start + 500 if end > len(items) { end = len(items) } ids := make([]int64, 0, end-start) byID := make(map[int64]*idWithTime, end-start) for i := start; i < end; i++ { if items[i].Id <= 0 { continue } ids = append(ids, items[i].Id) byID[items[i].Id] = &items[i] } if len(ids) == 0 { continue } chapters, err := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{ playletchaptermod.FN.PlayletChapter.PlayletId: bson.M{"$in": ids}, }, options.Find(). SetSort(bson.D{ {Key: playletchaptermod.FN.PlayletChapter.CurrentEpisode, Value: 1}, {Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1}, }). SetProjection(bson.M{ playletchaptermod.FN.PlayletChapter.Id: 1, playletchaptermod.FN.PlayletChapter.PlayletId: 1, playletchaptermod.FN.PlayletChapter.WatermarkVideoUrl: 1, })) if err != nil { return } seenFirst := make(map[int64]struct{}, len(ids)) for _, ch := range chapters { if ch == nil || ch.Id <= 0 { continue } item := byID[ch.PlayletId] if item == nil { continue } if _, ok := seenFirst[ch.PlayletId]; ok { continue } seenFirst[ch.PlayletId] = struct{}{} if skipOriginalChapterWithoutWatermark(item.IsOriginal, ch.WatermarkVideoUrl) { item.SkipPlaySitemap = true continue } item.ChapterID = ch.Id } } } func formatLastMod(t time.Time) string { if t.IsZero() { return "" } return t.Format(time.DateOnly) } func getPlayletList(ctx context.Context) listedPlaylets { result := listedPlaylets{ items: make([]idWithTime, 0), themeIDs: make(map[int64]struct{}), tagIDs: make(map[int64]struct{}), } filter := bson.M{ playletmod.FN.Playlet.Status: playletmod.StatusUp, } pageSize := int64(1000) for i := int64(0); i < 500; i++ { skip := i * pageSize list, _ := playletmod.FindManyPlaylet(ctx, filter, options.Find(). SetSort(bson.M{playletmod.FN.Playlet.Id: -1}). SetSkip(skip). SetLimit(pageSize). SetProjection(bson.M{ playletmod.FN.Playlet.Id: 1, playletmod.FN.Playlet.Slug: 1, playletmod.FN.Playlet.IsOriginal: 1, playletmod.FN.Playlet.UpdatedAt: 1, playletmod.FN.Playlet.OnlineAt: 1, playletmod.FN.Playlet.ThemeIds: 1, playletmod.FN.Playlet.TagIds: 1, })) for _, info := range list { if info == nil || info.Id <= 0 { continue } lastMod := info.UpdatedAt if lastMod.IsZero() { lastMod = info.OnlineAt } result.items = append(result.items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod, IsOriginal: info.IsOriginal}) addListedIDs(result.themeIDs, info.ThemeIds) addListedIDs(result.tagIDs, info.TagIds) } if len(list) < int(pageSize) { break } } return result } func getThemeList(ctx context.Context) []idWithTime { filter := bson.M{ playletthememod.FN.PlayletTheme.Status: true, } pageSize := int64(1000) items := make([]idWithTime, 0) for i := int64(0); i < 100; i++ { skip := i * pageSize list, _ := playletthememod.FindManyPlayletTheme(ctx, filter, options.Find(). SetSort(bson.M{playletthememod.FN.PlayletTheme.Rank: -1}). SetSkip(skip). SetLimit(pageSize)) for _, info := range list { if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" { continue } items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}) } if len(list) < int(pageSize) { break } } return items } func getTagList(ctx context.Context) []idWithTime { filter := bson.M{ playlettagmod.FN.PlayletTag.Status: true, } pageSize := int64(1000) items := make([]idWithTime, 0) for i := int64(0); i < 100; i++ { skip := i * pageSize list, _ := playlettagmod.FindManyPlayletTag(ctx, filter, options.Find(). SetSort(bson.M{playlettagmod.FN.PlayletTag.Rank: -1}). SetSkip(skip). SetLimit(pageSize)) for _, info := range list { if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" { continue } items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}) } if len(list) < int(pageSize) { break } } return items }