rootandClaude Opus 4.7
85c0071fd8
来源：hgdjnet-server 9bfbafe10d417deea959bd53117bc88a10e60a83 项目身份 HKDJ / JHA-216，独立官网仓库，Jenkins 守卫键 hkdjnet 测试目录 /server/hkdj_net，App/Web API 30331/30332 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
487 lines
14 KiB
Go
487 lines
14 KiB
Go
package sitemap
|
||
|
||
import (
|
||
"context"
|
||
"hkdj-web-server/pub/models/homeconfigmod"
|
||
"hkdj-web-server/pub/models/playletchaptermod"
|
||
"hkdj-web-server/pub/models/playletmod"
|
||
"hkdj-web-server/pub/models/playlettagmod"
|
||
"hkdj-web-server/pub/models/playletthememod"
|
||
"hkdj-web-server/pub/models/videoinformationmod"
|
||
"hkdj-web-server/pub/models/videotopicmod"
|
||
"hkdj-web-server/pub/seo/h5path"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"go.mongodb.org/mongo-driver/bson"
|
||
"go.mongodb.org/mongo-driver/mongo/options"
|
||
)
|
||
|
||
type idWithTime struct {
|
||
Id int64
|
||
Slug string
|
||
LastMod time.Time
|
||
IsOriginal bool
|
||
ChapterID int64
|
||
SkipPlaySitemap bool
|
||
}
|
||
|
||
type staticPageEntry struct {
|
||
path string
|
||
changeFreq string
|
||
priority float32
|
||
}
|
||
|
||
func buildStaticURLs(webAddr string, entries []staticPageEntry) []URL {
|
||
urls := make([]URL, 0, len(entries))
|
||
now := time.Now().Format(time.DateOnly)
|
||
for _, entry := range entries {
|
||
urls = append(urls, URL{
|
||
Loc: h5path.SitemapAbs(webAddr, entry.path),
|
||
LastMod: now,
|
||
ChangeFreq: entry.changeFreq,
|
||
Priority: entry.priority,
|
||
})
|
||
}
|
||
return urls
|
||
}
|
||
|
||
func entityPath(prefix, slug string, id int64) string {
|
||
return h5path.Entity(prefix, slug, id)
|
||
}
|
||
|
||
func hgdjStaticPages() []staticPageEntry {
|
||
return []staticPageEntry{
|
||
{path: "/", changeFreq: "daily", priority: 1.0},
|
||
{path: h5path.DramaListPath, changeFreq: "daily", priority: 0.9},
|
||
{path: h5path.SeriesPath, changeFreq: "daily", priority: 0.9},
|
||
{path: h5path.CollectionsPath, changeFreq: "daily", priority: 0.8},
|
||
{path: h5path.RankingsPath, changeFreq: "daily", priority: 0.8},
|
||
{path: h5path.RankingsPath + "/2", changeFreq: "daily", priority: 0.7},
|
||
{path: h5path.RankingsPath + "/3", changeFreq: "daily", priority: 0.7},
|
||
{path: h5path.LatestPath, changeFreq: "daily", priority: 0.8},
|
||
{path: h5path.FeaturedPath, changeFreq: "daily", priority: 0.8},
|
||
{path: h5path.FeaturedPath + "/2", changeFreq: "daily", priority: 0.7},
|
||
{path: h5path.FeaturedPath + "/3", changeFreq: "daily", priority: 0.7},
|
||
{path: "/about", changeFreq: "yearly", priority: 0.3},
|
||
{path: "/help", changeFreq: "yearly", priority: 0.3},
|
||
{path: "/terms", changeFreq: "yearly", priority: 0.3},
|
||
{path: "/privacy", changeFreq: "yearly", priority: 0.3},
|
||
{path: "/2257", changeFreq: "yearly", priority: 0.3},
|
||
{path: "/dmca", changeFreq: "yearly", priority: 0.3},
|
||
}
|
||
}
|
||
|
||
// getURL 并发拉取短剧/视频数据并写入 sitemap 分片，URL 规则对齐 hkdj-web 语义页面路径。
|
||
func getURL(sg *SitemapGenerator) {
|
||
webAddr := strings.TrimSpace(sg.WebAddr)
|
||
if webAddr == "" {
|
||
return
|
||
}
|
||
|
||
ctx := context.Background()
|
||
var (
|
||
wg sync.WaitGroup
|
||
listed listedPlaylets
|
||
themeItems []idWithTime
|
||
tagItems []idWithTime
|
||
videos listedVideos
|
||
homeConfigs []homeConfigItem
|
||
)
|
||
|
||
wg.Add(5)
|
||
go func() {
|
||
defer wg.Done()
|
||
listed = getPlayletList(ctx)
|
||
}()
|
||
go func() {
|
||
defer wg.Done()
|
||
themeItems = getThemeList(ctx)
|
||
}()
|
||
go func() {
|
||
defer wg.Done()
|
||
tagItems = getTagList(ctx)
|
||
}()
|
||
go func() {
|
||
defer wg.Done()
|
||
videos = getVideoList(ctx)
|
||
}()
|
||
go func() {
|
||
defer wg.Done()
|
||
homeConfigs = getHomeConfigList(ctx)
|
||
}()
|
||
wg.Wait()
|
||
|
||
// 题材/标签/视频专题没有上架内容时不写入，避免空列表页被收录。
|
||
themeItems = keepWithListedPlaylets(themeItems, listed.themeIDs)
|
||
tagItems = keepWithListedPlaylets(tagItems, listed.tagIDs)
|
||
videoTopics := keepWithListedPlaylets(videos.topics, videos.topicIDs)
|
||
|
||
pageURLs := buildStaticURLs(webAddr, hgdjStaticPages())
|
||
for _, item := range themeItems {
|
||
pageURLs = append(pageURLs, newURL(webAddr, h5path.ListTheme(item.Slug, item.Id), item.LastMod, "daily", 0.8))
|
||
pageURLs = append(pageURLs, newURL(webAddr, h5path.TopicDetail(item.Slug, item.Id), item.LastMod, "daily", 0.8))
|
||
}
|
||
for _, item := range tagItems {
|
||
if path := h5path.Tag(item.Slug, item.Id); path != "" {
|
||
pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "weekly", 0.6))
|
||
}
|
||
}
|
||
for _, item := range videoTopics {
|
||
pageURLs = append(pageURLs, newURL(webAddr, h5path.SeriesTopic(item.Id), item.LastMod, "daily", 0.7))
|
||
}
|
||
for _, item := range homeConfigs {
|
||
path := ""
|
||
switch item.Type {
|
||
case homeconfigmod.TypePlaylet:
|
||
path = h5path.MoreList(item.Slug, item.Id)
|
||
case homeconfigmod.TypeVideo:
|
||
path = h5path.MoreSeriesList(item.Slug, item.Id)
|
||
}
|
||
if path != "" {
|
||
pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "daily", 0.7))
|
||
}
|
||
}
|
||
if len(pageURLs) > 0 {
|
||
_ = sg.writeSitemapFile("sitemap-pages.xml", pageURLs)
|
||
}
|
||
|
||
// 各集 /drama/{slug}/{ep} 由前端 noindex；sitemap 只保留 /drama/{slug}。
|
||
// 原创且第一集无水印的播放入口不进 sitemap，避免空壳播放页被提交。
|
||
populateFirstChapters(ctx, listed.items)
|
||
if urls := buildPlayletPathURLs(webAddr, listed.items, "weekly", 0.7); len(urls) > 0 {
|
||
_ = sg.writeSitemapFiles("sitemap-playletdetail", urls)
|
||
}
|
||
if urls := buildVideoPlayURLs(webAddr, videos.items, "weekly", 0.6); len(urls) > 0 {
|
||
_ = sg.writeSitemapFiles("sitemap-seriesplay", urls)
|
||
}
|
||
}
|
||
|
||
type listedPlaylets struct {
|
||
items []idWithTime
|
||
themeIDs map[int64]struct{}
|
||
tagIDs map[int64]struct{}
|
||
}
|
||
|
||
type listedVideos struct {
|
||
items []idWithTime
|
||
topics []idWithTime
|
||
topicIDs map[int64]struct{}
|
||
}
|
||
|
||
type homeConfigItem struct {
|
||
Id int64
|
||
Slug string
|
||
Type int32
|
||
LastMod time.Time
|
||
}
|
||
|
||
func newURL(webAddr, path string, lastMod time.Time, changeFreq string, priority float32) URL {
|
||
mod := formatLastMod(lastMod)
|
||
if mod == "" {
|
||
mod = time.Now().Format(time.DateOnly)
|
||
}
|
||
return URL{
|
||
Loc: h5path.SitemapAbs(webAddr, path),
|
||
LastMod: mod,
|
||
ChangeFreq: changeFreq,
|
||
Priority: priority,
|
||
}
|
||
}
|
||
|
||
func keepWithListedPlaylets(items []idWithTime, listed map[int64]struct{}) []idWithTime {
|
||
if len(items) == 0 || len(listed) == 0 {
|
||
return nil
|
||
}
|
||
out := make([]idWithTime, 0, len(items))
|
||
for _, item := range items {
|
||
if _, ok := listed[item.Id]; ok {
|
||
out = append(out, item)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func addListedIDs(dst map[int64]struct{}, ids []int64) {
|
||
for _, id := range ids {
|
||
if id > 0 {
|
||
dst[id] = struct{}{}
|
||
}
|
||
}
|
||
}
|
||
|
||
func buildPlayletPathURLs(webAddr string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||
urls := make([]URL, 0, len(items))
|
||
for _, item := range items {
|
||
if item.SkipPlaySitemap {
|
||
continue
|
||
}
|
||
path := h5path.Playlet(item.Slug, item.Id)
|
||
if path == "" {
|
||
continue
|
||
}
|
||
urls = append(urls, newURL(webAddr, path, item.LastMod, changeFreq, priority))
|
||
}
|
||
return urls
|
||
}
|
||
|
||
func buildVideoPlayURLs(webAddr string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||
urls := make([]URL, 0, len(items))
|
||
for _, item := range items {
|
||
path := h5path.VideoPlay(item.Slug, item.Id)
|
||
if path == "" {
|
||
continue
|
||
}
|
||
urls = append(urls, newURL(webAddr, path, item.LastMod, changeFreq, priority))
|
||
}
|
||
return urls
|
||
}
|
||
|
||
func playletIDs(items []idWithTime) []int64 {
|
||
ids := make([]int64, 0, len(items))
|
||
for _, item := range items {
|
||
if item.Id > 0 {
|
||
ids = append(ids, item.Id)
|
||
}
|
||
}
|
||
return ids
|
||
}
|
||
|
||
func skipOriginalChapterWithoutWatermark(isOriginal bool, watermarkURL string) bool {
|
||
return isOriginal && strings.TrimSpace(watermarkURL) == ""
|
||
}
|
||
|
||
// populateFirstChapters 每剧只看第一集；原创且无水印则不写播放 URL。
|
||
func populateFirstChapters(ctx context.Context, items []idWithTime) {
|
||
for start := 0; start < len(items); start += 500 {
|
||
end := start + 500
|
||
if end > len(items) {
|
||
end = len(items)
|
||
}
|
||
ids := make([]int64, 0, end-start)
|
||
byID := make(map[int64]*idWithTime, end-start)
|
||
for i := start; i < end; i++ {
|
||
if items[i].Id <= 0 {
|
||
continue
|
||
}
|
||
ids = append(ids, items[i].Id)
|
||
byID[items[i].Id] = &items[i]
|
||
}
|
||
if len(ids) == 0 {
|
||
continue
|
||
}
|
||
chapters, err := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{
|
||
playletchaptermod.FN.PlayletChapter.PlayletId: bson.M{"$in": ids},
|
||
}, options.Find().
|
||
SetSort(bson.D{
|
||
{Key: playletchaptermod.FN.PlayletChapter.CurrentEpisode, Value: 1},
|
||
{Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1},
|
||
}).
|
||
SetProjection(bson.M{
|
||
playletchaptermod.FN.PlayletChapter.Id: 1,
|
||
playletchaptermod.FN.PlayletChapter.PlayletId: 1,
|
||
playletchaptermod.FN.PlayletChapter.WatermarkVideoUrl: 1,
|
||
}))
|
||
if err != nil {
|
||
return
|
||
}
|
||
seenFirst := make(map[int64]struct{}, len(ids))
|
||
for _, ch := range chapters {
|
||
if ch == nil || ch.Id <= 0 {
|
||
continue
|
||
}
|
||
item := byID[ch.PlayletId]
|
||
if item == nil {
|
||
continue
|
||
}
|
||
if _, ok := seenFirst[ch.PlayletId]; ok {
|
||
continue
|
||
}
|
||
seenFirst[ch.PlayletId] = struct{}{}
|
||
if skipOriginalChapterWithoutWatermark(item.IsOriginal, ch.WatermarkVideoUrl) {
|
||
item.SkipPlaySitemap = true
|
||
continue
|
||
}
|
||
item.ChapterID = ch.Id
|
||
}
|
||
}
|
||
}
|
||
|
||
func formatLastMod(t time.Time) string {
|
||
if t.IsZero() {
|
||
return ""
|
||
}
|
||
return t.Format(time.DateOnly)
|
||
}
|
||
|
||
func getPlayletList(ctx context.Context) listedPlaylets {
|
||
result := listedPlaylets{
|
||
items: make([]idWithTime, 0),
|
||
themeIDs: make(map[int64]struct{}),
|
||
tagIDs: make(map[int64]struct{}),
|
||
}
|
||
filter := bson.M{
|
||
playletmod.FN.Playlet.Status: playletmod.StatusUp,
|
||
}
|
||
pageSize := int64(1000)
|
||
for i := int64(0); i < 500; i++ {
|
||
skip := i * pageSize
|
||
list, _ := playletmod.FindManyPlaylet(ctx, filter, options.Find().
|
||
SetSort(bson.M{playletmod.FN.Playlet.Id: -1}).
|
||
SetSkip(skip).
|
||
SetLimit(pageSize).
|
||
SetProjection(bson.M{
|
||
playletmod.FN.Playlet.Id: 1,
|
||
playletmod.FN.Playlet.Slug: 1,
|
||
playletmod.FN.Playlet.IsOriginal: 1,
|
||
playletmod.FN.Playlet.UpdatedAt: 1,
|
||
playletmod.FN.Playlet.OnlineAt: 1,
|
||
playletmod.FN.Playlet.ThemeIds: 1,
|
||
playletmod.FN.Playlet.TagIds: 1,
|
||
}))
|
||
for _, info := range list {
|
||
if info == nil || info.Id <= 0 {
|
||
continue
|
||
}
|
||
lastMod := info.UpdatedAt
|
||
if lastMod.IsZero() {
|
||
lastMod = info.OnlineAt
|
||
}
|
||
result.items = append(result.items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod, IsOriginal: info.IsOriginal})
|
||
addListedIDs(result.themeIDs, info.ThemeIds)
|
||
addListedIDs(result.tagIDs, info.TagIds)
|
||
}
|
||
if len(list) < int(pageSize) {
|
||
break
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func getThemeList(ctx context.Context) []idWithTime {
|
||
filter := bson.M{
|
||
playletthememod.FN.PlayletTheme.Status: true,
|
||
}
|
||
pageSize := int64(1000)
|
||
items := make([]idWithTime, 0)
|
||
for i := int64(0); i < 100; i++ {
|
||
skip := i * pageSize
|
||
list, _ := playletthememod.FindManyPlayletTheme(ctx, filter, options.Find().
|
||
SetSort(bson.M{playletthememod.FN.PlayletTheme.Rank: -1}).
|
||
SetSkip(skip).
|
||
SetLimit(pageSize))
|
||
for _, info := range list {
|
||
if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" {
|
||
continue
|
||
}
|
||
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
||
}
|
||
if len(list) < int(pageSize) {
|
||
break
|
||
}
|
||
}
|
||
return items
|
||
}
|
||
|
||
func getTagList(ctx context.Context) []idWithTime {
|
||
filter := bson.M{
|
||
playlettagmod.FN.PlayletTag.Status: true,
|
||
}
|
||
pageSize := int64(1000)
|
||
items := make([]idWithTime, 0)
|
||
for i := int64(0); i < 100; i++ {
|
||
skip := i * pageSize
|
||
list, _ := playlettagmod.FindManyPlayletTag(ctx, filter, options.Find().
|
||
SetSort(bson.M{playlettagmod.FN.PlayletTag.Rank: -1}).
|
||
SetSkip(skip).
|
||
SetLimit(pageSize))
|
||
for _, info := range list {
|
||
if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" {
|
||
continue
|
||
}
|
||
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
||
}
|
||
if len(list) < int(pageSize) {
|
||
break
|
||
}
|
||
}
|
||
return items
|
||
}
|
||
|
||
func getVideoList(ctx context.Context) listedVideos {
|
||
result := listedVideos{
|
||
items: make([]idWithTime, 0),
|
||
topics: make([]idWithTime, 0),
|
||
topicIDs: make(map[int64]struct{}),
|
||
}
|
||
filter := bson.M{
|
||
videoinformationmod.FN.VideoInformation.Status: videoinformationmod.InformationSatusVerifyOK,
|
||
}
|
||
pageSize := int64(1000)
|
||
for i := int64(0); i < 500; i++ {
|
||
skip := i * pageSize
|
||
list, _ := videoinformationmod.FindManyVideoInformations(ctx, filter, options.Find().
|
||
SetSort(bson.M{videoinformationmod.FN.VideoInformation.Id: -1}).
|
||
SetSkip(skip).
|
||
SetLimit(pageSize).
|
||
SetProjection(bson.M{
|
||
videoinformationmod.FN.VideoInformation.Id: 1,
|
||
videoinformationmod.FN.VideoInformation.Slug: 1,
|
||
videoinformationmod.FN.VideoInformation.UpdatedAt: 1,
|
||
videoinformationmod.FN.VideoInformation.AddedTime: 1,
|
||
videoinformationmod.FN.VideoInformation.TopicIds: 1,
|
||
}))
|
||
for _, info := range list {
|
||
if info == nil || info.Id <= 0 {
|
||
continue
|
||
}
|
||
lastMod := info.UpdatedAt
|
||
if lastMod.IsZero() {
|
||
lastMod = info.AddedTime
|
||
}
|
||
result.items = append(result.items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod})
|
||
addListedIDs(result.topicIDs, info.TopicIds)
|
||
}
|
||
if len(list) < int(pageSize) {
|
||
break
|
||
}
|
||
}
|
||
|
||
topicFilter := bson.M{videotopicmod.FN.VideoTopicModel.Status: true}
|
||
for i := int64(0); i < 100; i++ {
|
||
skip := i * pageSize
|
||
list, _ := videotopicmod.FindManyVideoTopicModels(ctx, topicFilter, options.Find().
|
||
SetSort(bson.M{videotopicmod.FN.VideoTopicModel.Rank: -1}).
|
||
SetSkip(skip).
|
||
SetLimit(pageSize))
|
||
for _, info := range list {
|
||
if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" {
|
||
continue
|
||
}
|
||
result.topics = append(result.topics, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
||
}
|
||
if len(list) < int(pageSize) {
|
||
break
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func getHomeConfigList(ctx context.Context) []homeConfigItem {
|
||
list, _ := homeconfigmod.FindManyHomeConfig(ctx, bson.M{
|
||
homeconfigmod.FN.HomeConfig.Status: true,
|
||
homeconfigmod.FN.HomeConfig.Type: bson.M{
|
||
"$in": []int32{homeconfigmod.TypePlaylet, homeconfigmod.TypeVideo},
|
||
},
|
||
}, options.Find().SetSort(bson.M{homeconfigmod.FN.HomeConfig.Rank: -1}))
|
||
items := make([]homeConfigItem, 0, len(list))
|
||
for _, info := range list {
|
||
if info == nil || info.Id <= 0 {
|
||
continue
|
||
}
|
||
items = append(items, homeConfigItem{Id: info.Id, Slug: info.Slug, Type: info.Type, LastMod: info.UpdatedAt})
|
||
}
|
||
return items
|
||
}
|