init: import web_huangguo2 server project

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-09-29 10:47:38 +08:00
co-authored by Claude Opus 4.7
commit a50c9c342e
668 changed files with 141008 additions and 0 deletions
+486
View File
@@ -0,0 +1,486 @@
package sitemap
import (
"context"
"strings"
"sync"
"time"
"videoweb/pub/models/homeconfigmod"
"videoweb/pub/models/playletchaptermod"
"videoweb/pub/models/playletmod"
"videoweb/pub/models/playlettagmod"
"videoweb/pub/models/playletthememod"
"videoweb/pub/models/videoinformationmod"
"videoweb/pub/models/videotopicmod"
"videoweb/pub/seo/h5path"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
type idWithTime struct {
Id int64
Slug string
LastMod time.Time
IsOriginal bool
ChapterID int64
SkipPlaySitemap bool
}
type staticPageEntry struct {
path string
changeFreq string
priority float32
}
func buildStaticURLs(webAddr string, entries []staticPageEntry) []URL {
urls := make([]URL, 0, len(entries))
now := time.Now().Format(time.DateOnly)
for _, entry := range entries {
urls = append(urls, URL{
Loc: h5path.Abs(webAddr, entry.path),
LastMod: now,
ChangeFreq: entry.changeFreq,
Priority: entry.priority,
})
}
return urls
}
func entityPath(prefix, slug string, id int64) string {
return h5path.Entity(prefix, slug, id)
}
func hgdjStaticPages() []staticPageEntry {
return []staticPageEntry{
{path: "/", changeFreq: "daily", priority: 1.0},
{path: "/list", changeFreq: "daily", priority: 0.9},
{path: "/series", changeFreq: "daily", priority: 0.9},
{path: "/topic", changeFreq: "daily", priority: 0.8},
{path: "/rank", changeFreq: "daily", priority: 0.8},
{path: "/rank/2", changeFreq: "daily", priority: 0.7},
{path: "/rank/3", changeFreq: "daily", priority: 0.7},
{path: "/recently-added", changeFreq: "daily", priority: 0.8},
{path: "/popular-recommend", changeFreq: "daily", priority: 0.8},
{path: "/popular-recommend/2", changeFreq: "daily", priority: 0.7},
{path: "/popular-recommend/3", changeFreq: "daily", priority: 0.7},
{path: "/about", changeFreq: "yearly", priority: 0.3},
{path: "/help", changeFreq: "yearly", priority: 0.3},
{path: "/terms", changeFreq: "yearly", priority: 0.3},
{path: "/privacy", changeFreq: "yearly", priority: 0.3},
{path: "/2257", changeFreq: "yearly", priority: 0.3},
{path: "/dmca", changeFreq: "yearly", priority: 0.3},
}
}
// getURL 并发拉取短剧/视频数据并写入 sitemap 分片,URL 规则对齐 jha-1045-hgdjnet-web。
func getURL(sg *SitemapGenerator) {
webAddr := strings.TrimSpace(sg.WebAddr)
if webAddr == "" {
return
}
ctx := context.Background()
var (
wg sync.WaitGroup
listed listedPlaylets
themeItems []idWithTime
tagItems []idWithTime
videos listedVideos
homeConfigs []homeConfigItem
)
wg.Add(5)
go func() {
defer wg.Done()
listed = getPlayletList(ctx)
}()
go func() {
defer wg.Done()
themeItems = getThemeList(ctx)
}()
go func() {
defer wg.Done()
tagItems = getTagList(ctx)
}()
go func() {
defer wg.Done()
videos = getVideoList(ctx)
}()
go func() {
defer wg.Done()
homeConfigs = getHomeConfigList(ctx)
}()
wg.Wait()
// 题材/标签/视频专题没有上架内容时不写入,避免空列表页被收录。
themeItems = keepWithListedPlaylets(themeItems, listed.themeIDs)
tagItems = keepWithListedPlaylets(tagItems, listed.tagIDs)
videoTopics := keepWithListedPlaylets(videos.topics, videos.topicIDs)
pageURLs := buildStaticURLs(webAddr, hgdjStaticPages())
for _, item := range themeItems {
pageURLs = append(pageURLs, newURL(webAddr, h5path.ListTheme(item.Slug, item.Id), item.LastMod, "daily", 0.8))
pageURLs = append(pageURLs, newURL(webAddr, h5path.TopicDetail(item.Slug, item.Id), item.LastMod, "daily", 0.8))
}
for _, item := range tagItems {
if path := h5path.Tag(item.Slug, item.Id); path != "" {
pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "weekly", 0.6))
}
}
for _, item := range videoTopics {
pageURLs = append(pageURLs, newURL(webAddr, h5path.SeriesTopic(item.Id), item.LastMod, "daily", 0.7))
}
for _, item := range homeConfigs {
path := ""
switch item.Type {
case homeconfigmod.TypePlaylet:
path = h5path.MoreList(item.Slug, item.Id)
case homeconfigmod.TypeVideo:
path = h5path.MoreSeriesList(item.Slug, item.Id)
}
if path != "" {
pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "daily", 0.7))
}
}
if len(pageURLs) > 0 {
_ = sg.writeSitemapFile("sitemap-pages.xml", pageURLs)
}
// 各集 /play/{slug}/{ep} 由前端 noindex;sitemap 只保留 /play/{slug}。
// 原创且第一集无水印的播放入口不进 sitemap,避免空壳播放页被提交。
populateFirstChapters(ctx, listed.items)
if urls := buildPlayletPathURLs(webAddr, listed.items, "weekly", 0.7); len(urls) > 0 {
_ = sg.writeSitemapFiles("sitemap-playletdetail", urls)
}
if urls := buildVideoPlayURLs(webAddr, videos.items, "weekly", 0.6); len(urls) > 0 {
_ = sg.writeSitemapFiles("sitemap-seriesplay", urls)
}
}
type listedPlaylets struct {
items []idWithTime
themeIDs map[int64]struct{}
tagIDs map[int64]struct{}
}
type listedVideos struct {
items []idWithTime
topics []idWithTime
topicIDs map[int64]struct{}
}
type homeConfigItem struct {
Id int64
Slug string
Type int32
LastMod time.Time
}
func newURL(webAddr, path string, lastMod time.Time, changeFreq string, priority float32) URL {
mod := formatLastMod(lastMod)
if mod == "" {
mod = time.Now().Format(time.DateOnly)
}
return URL{
Loc: h5path.Abs(webAddr, path),
LastMod: mod,
ChangeFreq: changeFreq,
Priority: priority,
}
}
func keepWithListedPlaylets(items []idWithTime, listed map[int64]struct{}) []idWithTime {
if len(items) == 0 || len(listed) == 0 {
return nil
}
out := make([]idWithTime, 0, len(items))
for _, item := range items {
if _, ok := listed[item.Id]; ok {
out = append(out, item)
}
}
return out
}
func addListedIDs(dst map[int64]struct{}, ids []int64) {
for _, id := range ids {
if id > 0 {
dst[id] = struct{}{}
}
}
}
func buildPlayletPathURLs(webAddr string, items []idWithTime, changeFreq string, priority float32) []URL {
urls := make([]URL, 0, len(items))
for _, item := range items {
if item.SkipPlaySitemap {
continue
}
path := h5path.Playlet(item.Slug, item.Id)
if path == "" {
continue
}
urls = append(urls, newURL(webAddr, path, item.LastMod, changeFreq, priority))
}
return urls
}
func buildVideoPlayURLs(webAddr string, items []idWithTime, changeFreq string, priority float32) []URL {
urls := make([]URL, 0, len(items))
for _, item := range items {
path := h5path.VideoPlay(item.Slug, item.Id)
if path == "" {
continue
}
urls = append(urls, newURL(webAddr, path, item.LastMod, changeFreq, priority))
}
return urls
}
func playletIDs(items []idWithTime) []int64 {
ids := make([]int64, 0, len(items))
for _, item := range items {
if item.Id > 0 {
ids = append(ids, item.Id)
}
}
return ids
}
func skipOriginalChapterWithoutWatermark(isOriginal bool, watermarkURL string) bool {
return isOriginal && strings.TrimSpace(watermarkURL) == ""
}
// populateFirstChapters 每剧只看第一集;原创且无水印则不写播放 URL。
func populateFirstChapters(ctx context.Context, items []idWithTime) {
for start := 0; start < len(items); start += 500 {
end := start + 500
if end > len(items) {
end = len(items)
}
ids := make([]int64, 0, end-start)
byID := make(map[int64]*idWithTime, end-start)
for i := start; i < end; i++ {
if items[i].Id <= 0 {
continue
}
ids = append(ids, items[i].Id)
byID[items[i].Id] = &items[i]
}
if len(ids) == 0 {
continue
}
chapters, err := playletchaptermod.FindManyPlayletChapter(ctx, bson.M{
playletchaptermod.FN.PlayletChapter.PlayletId: bson.M{"$in": ids},
}, options.Find().
SetSort(bson.D{
{Key: playletchaptermod.FN.PlayletChapter.CurrentEpisode, Value: 1},
{Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1},
}).
SetProjection(bson.M{
playletchaptermod.FN.PlayletChapter.Id: 1,
playletchaptermod.FN.PlayletChapter.PlayletId: 1,
playletchaptermod.FN.PlayletChapter.WatermarkVideoUrl: 1,
}))
if err != nil {
return
}
seenFirst := make(map[int64]struct{}, len(ids))
for _, ch := range chapters {
if ch == nil || ch.Id <= 0 {
continue
}
item := byID[ch.PlayletId]
if item == nil {
continue
}
if _, ok := seenFirst[ch.PlayletId]; ok {
continue
}
seenFirst[ch.PlayletId] = struct{}{}
if skipOriginalChapterWithoutWatermark(item.IsOriginal, ch.WatermarkVideoUrl) {
item.SkipPlaySitemap = true
continue
}
item.ChapterID = ch.Id
}
}
}
func formatLastMod(t time.Time) string {
if t.IsZero() {
return ""
}
return t.Format(time.DateOnly)
}
func getPlayletList(ctx context.Context) listedPlaylets {
result := listedPlaylets{
items: make([]idWithTime, 0),
themeIDs: make(map[int64]struct{}),
tagIDs: make(map[int64]struct{}),
}
filter := bson.M{
playletmod.FN.Playlet.Status: playletmod.StatusUp,
}
pageSize := int64(1000)
for i := int64(0); i < 500; i++ {
skip := i * pageSize
list, _ := playletmod.FindManyPlaylet(ctx, filter, options.Find().
SetSort(bson.M{playletmod.FN.Playlet.Id: -1}).
SetSkip(skip).
SetLimit(pageSize).
SetProjection(bson.M{
playletmod.FN.Playlet.Id: 1,
playletmod.FN.Playlet.Slug: 1,
playletmod.FN.Playlet.IsOriginal: 1,
playletmod.FN.Playlet.UpdatedAt: 1,
playletmod.FN.Playlet.OnlineAt: 1,
playletmod.FN.Playlet.ThemeIds: 1,
playletmod.FN.Playlet.TagIds: 1,
}))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
lastMod := info.UpdatedAt
if lastMod.IsZero() {
lastMod = info.OnlineAt
}
result.items = append(result.items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod, IsOriginal: info.IsOriginal})
addListedIDs(result.themeIDs, info.ThemeIds)
addListedIDs(result.tagIDs, info.TagIds)
}
if len(list) < int(pageSize) {
break
}
}
return result
}
func getThemeList(ctx context.Context) []idWithTime {
filter := bson.M{
playletthememod.FN.PlayletTheme.Status: true,
}
pageSize := int64(1000)
items := make([]idWithTime, 0)
for i := int64(0); i < 100; i++ {
skip := i * pageSize
list, _ := playletthememod.FindManyPlayletTheme(ctx, filter, options.Find().
SetSort(bson.M{playletthememod.FN.PlayletTheme.Rank: -1}).
SetSkip(skip).
SetLimit(pageSize))
for _, info := range list {
if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
if len(list) < int(pageSize) {
break
}
}
return items
}
func getTagList(ctx context.Context) []idWithTime {
filter := bson.M{
playlettagmod.FN.PlayletTag.Status: true,
}
pageSize := int64(1000)
items := make([]idWithTime, 0)
for i := int64(0); i < 100; i++ {
skip := i * pageSize
list, _ := playlettagmod.FindManyPlayletTag(ctx, filter, options.Find().
SetSort(bson.M{playlettagmod.FN.PlayletTag.Rank: -1}).
SetSkip(skip).
SetLimit(pageSize))
for _, info := range list {
if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
if len(list) < int(pageSize) {
break
}
}
return items
}
func getVideoList(ctx context.Context) listedVideos {
result := listedVideos{
items: make([]idWithTime, 0),
topics: make([]idWithTime, 0),
topicIDs: make(map[int64]struct{}),
}
filter := bson.M{
videoinformationmod.FN.VideoInformation.Status: videoinformationmod.InformationSatusVerifyOK,
}
pageSize := int64(1000)
for i := int64(0); i < 500; i++ {
skip := i * pageSize
list, _ := videoinformationmod.FindManyVideoInformations(ctx, filter, options.Find().
SetSort(bson.M{videoinformationmod.FN.VideoInformation.Id: -1}).
SetSkip(skip).
SetLimit(pageSize).
SetProjection(bson.M{
videoinformationmod.FN.VideoInformation.Id: 1,
videoinformationmod.FN.VideoInformation.Slug: 1,
videoinformationmod.FN.VideoInformation.UpdatedAt: 1,
videoinformationmod.FN.VideoInformation.AddedTime: 1,
videoinformationmod.FN.VideoInformation.TopicIds: 1,
}))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
lastMod := info.UpdatedAt
if lastMod.IsZero() {
lastMod = info.AddedTime
}
result.items = append(result.items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod})
addListedIDs(result.topicIDs, info.TopicIds)
}
if len(list) < int(pageSize) {
break
}
}
topicFilter := bson.M{videotopicmod.FN.VideoTopicModel.Status: true}
for i := int64(0); i < 100; i++ {
skip := i * pageSize
list, _ := videotopicmod.FindManyVideoTopicModels(ctx, topicFilter, options.Find().
SetSort(bson.M{videotopicmod.FN.VideoTopicModel.Rank: -1}).
SetSkip(skip).
SetLimit(pageSize))
for _, info := range list {
if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" {
continue
}
result.topics = append(result.topics, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
if len(list) < int(pageSize) {
break
}
}
return result
}
func getHomeConfigList(ctx context.Context) []homeConfigItem {
list, _ := homeconfigmod.FindManyHomeConfig(ctx, bson.M{
homeconfigmod.FN.HomeConfig.Status: true,
homeconfigmod.FN.HomeConfig.Type: bson.M{
"$in": []int32{homeconfigmod.TypePlaylet, homeconfigmod.TypeVideo},
},
}, options.Find().SetSort(bson.M{homeconfigmod.FN.HomeConfig.Rank: -1}))
items := make([]homeConfigItem, 0, len(list))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
items = append(items, homeConfigItem{Id: info.Id, Slug: info.Slug, Type: info.Type, LastMod: info.UpdatedAt})
}
return items
}