Initial commit: web_hghl_server

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-10-26 16:28:14 +08:00
co-authored by Claude Opus 4.7
commit 65fca60622
653 changed files with 135901 additions and 0 deletions
+312
View File
@@ -0,0 +1,312 @@
package sitemap
import (
"context"
"fmt"
"net/url"
"strconv"
"strings"
"sync"
"time"
"videoweb/pub/common/utils"
"videoweb/pub/models/actress"
"videoweb/pub/models/cfg"
"videoweb/pub/models/translationcategorymod"
"videoweb/pub/models/translationmod"
"videoweb/pub/models/videocategorymod"
"videoweb/pub/models/videoinformationmod"
"videoweb/pub/pubdef"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
// idWithTime 携带 ID、slug 和实际更新时间
type idWithTime struct {
Id int64
Slug string
LastMod time.Time
}
func hasAnyID(ids []int64, used map[int64]struct{}) bool {
for _, id := range ids {
if _, ok := used[id]; ok {
return true
}
}
return false
}
func isSitemapChannelPath(path, prefix string, extras ...string) bool {
if path == prefix || strings.HasPrefix(path, prefix+"/") {
return true
}
for _, extra := range extras {
if path == extra {
return true
}
}
return false
}
func detailSegment(slug string, id int64) string {
if s := strings.TrimSpace(slug); s != "" {
return s
}
if id > 0 {
return strconv.FormatInt(id, 10)
}
return ""
}
func buildChannelKeyURLs(webAddr, channel, keyPrefix string, items []idWithTime, changeFreq string, priority float32) []URL {
urls := make([]URL, 0, len(items))
channel = strings.TrimRight(channel, "/")
for _, item := range items {
seg := detailSegment(item.Slug, item.Id)
if seg == "" {
continue
}
lastMod := item.LastMod.Format(time.DateOnly)
if item.LastMod.IsZero() {
lastMod = time.Now().Format(time.DateOnly)
}
urls = append(urls, URL{
Loc: utils.JoinUrlPathKeepSlash(webAddr, channel+"/"+keyPrefix+seg),
LastMod: lastMod,
ChangeFreq: changeFreq,
Priority: priority,
})
}
return urls
}
// buildCategoryURLs 分类页:/{channel}/c-{slug},无 slug 时回退 c-{id}。
func buildCategoryURLs(webAddr, channel string, items []idWithTime, changeFreq string, priority float32) []URL {
return buildChannelKeyURLs(webAddr, channel, "c-", items, changeFreq, priority)
}
// buildTagURLs 标签页:/{channel}/t-{slug},无 slug 时回退 t-{id}。
func buildTagURLs(webAddr, channel string, items []idWithTime, changeFreq string, priority float32) []URL {
return buildChannelKeyURLs(webAddr, channel, "t-", items, changeFreq, priority)
}
// buildDetailURLs 详情页:/{prefix}/{slug},无 slug 时回退数字 id。
func buildDetailURLs(webAddr, prefix string, items []idWithTime, changeFreq string, priority float32) []URL {
urls := make([]URL, 0, len(items))
for _, item := range items {
seg := detailSegment(item.Slug, item.Id)
if seg == "" {
continue
}
lastMod := item.LastMod.Format(time.DateOnly)
if item.LastMod.IsZero() {
lastMod = time.Now().Format(time.DateOnly)
}
urls = append(urls, URL{
Loc: utils.JoinUrlPathKeepSlash(webAddr, prefix+"/"+seg),
LastMod: lastMod,
ChangeFreq: changeFreq,
Priority: priority,
})
}
return urls
}
// buildURLs 根据 ID+时间列表批量构建 URL 条目
func buildURLs(webAddr string, pathFmt string, items []idWithTime, changeFreq string, priority float32) []URL {
urls := make([]URL, 0, len(items))
for _, item := range items {
lastMod := item.LastMod.Format(time.DateOnly)
if item.LastMod.IsZero() {
lastMod = time.Now().Format(time.DateOnly)
}
urls = append(urls, URL{
Loc: utils.JoinUrlPathKeepSlash(webAddr, fmt.Sprintf(pathFmt, item.Id)),
LastMod: lastMod,
ChangeFreq: changeFreq,
Priority: priority,
})
}
return urls
}
// filterIdsWithTime 返回 source 中存在于 validMap 的条目(含 slug)。
func filterIdsWithTime(source map[int64]struct{}, validMap map[int64]idWithTime) []idWithTime {
result := make([]idWithTime, 0, len(source))
for id := range source {
if item, ok := validMap[id]; ok {
result = append(result, item)
}
}
return result
}
func getVideoCategoryList(ctx context.Context, usedTagIds map[int64]struct{}, hasVideos bool) []idWithTime {
filter := bson.M{
videocategorymod.FN.VideoCategory.Status: true,
}
pageSize := int64(1000)
items := make([]idWithTime, 0)
for i := int64(0); i < 500; i++ {
skip := i * pageSize
list, _ := videocategorymod.FindManyVideoCategorys(ctx, filter, options.Find().SetSort(bson.M{videocategorymod.FN.VideoCategory.Id: -1}).SetSkip(skip).SetLimit(pageSize))
for _, info := range list {
if len(info.Tags) == 0 {
if !hasVideos {
continue
}
} else if !hasAnyID(info.Tags, usedTagIds) {
continue
}
if !videoCategoryHasContent(ctx, info) {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
if len(list) < int(pageSize) {
break
}
}
return items
}
// videoCategoryContentFilter 与 app QueryVideoMore 使用相同的视频筛选条件。
func videoCategoryContentFilter(category *videocategorymod.VideoCategory) bson.M {
filter := bson.M{
videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK,
}
if category != nil && len(category.Tags) > 0 {
filter[videoinformationmod.FN.VideoInformation.Tags] = bson.M{"$in": category.Tags}
}
return filter
}
func videoCategoryHasContent(ctx context.Context, category *videocategorymod.VideoCategory) bool {
if category == nil || !category.Status {
return false
}
count, err := videoinformationmod.CountVideoInformations(ctx, videoCategoryContentFilter(category))
return err == nil && count > 0
}
func getActress(ctx context.Context) map[int64]idWithTime {
filter := bson.M{
actress.FN.Actress.Status: true,
}
pageSize := int64(1000)
pageIds := make(map[int64]idWithTime)
for i := int64(0); i < 100; i++ {
skip := i * pageSize
list, _ := actress.FindManyActresss(ctx, filter, options.Find().SetSkip(skip).SetLimit(pageSize).SetSort(bson.M{actress.FN.Actress.Id: -1}))
for _, info := range list {
pageIds[info.Id] = idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}
}
if len(list) < int(pageSize) {
break
}
}
return pageIds
}
func videoEntriesToIDWithTime(entries []videoEntry) []idWithTime {
items := make([]idWithTime, len(entries))
for i, e := range entries {
items[i] = idWithTime{Id: e.Id, Slug: e.Slug, LastMod: e.LastMod}
}
return items
}
func getVideoList(ctx context.Context) ([]videoEntry, map[int64]struct{}, map[int64]struct{}) {
filter := bson.M{videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK}
pageSize := int64(1000)
items := make([]videoEntry, 0)
tagIds := make(map[int64]struct{})
actressIds := make(map[int64]struct{})
for i := int64(0); i < 500; i++ {
skip := i * pageSize
list, _ := videoinformationmod.FindManyVideoInformations(ctx, filter, options.Find().SetSort(bson.M{videoinformationmod.FN.VideoInformation.Id: -1}).SetLimit(pageSize).SetSkip(skip))
for _, info := range list {
items = append(items, videoEntry{
Id: info.Id,
Slug: info.Slug,
Title: info.Title,
LastMod: info.UpdatedAt,
UnencryptedPreMp4: info.Video.UnencryptedPreMp4,
})
for _, tagId := range info.Tags {
tagIds[tagId] = struct{}{}
}
for _, actressId := range info.ActressIds {
if actressId > 0 {
actressIds[actressId] = struct{}{}
}
}
}
if len(list) < int(pageSize) {
break
}
}
return items, tagIds, actressIds
}
func getTranslation(config cfg.SysCfg, ctx context.Context) []string {
urlList := make([]string, 0)
wg := &sync.WaitGroup{}
wg.Add(2)
var translationCategoryList translationcategorymod.TranslationCategoryList
var translationList translationmod.TranslationInfoList
go func() {
defer wg.Done()
categoryFilter := bson.M{
translationcategorymod.FN.TranslationCategory.Status: true,
}
categoryOpt := &options.FindOptions{}
categoryOpt.SetLimit(1000).SetSort(bson.M{translationcategorymod.FN.TranslationCategory.Rank: -1})
translationCategoryList, _ = translationcategorymod.FindManyTranslationCategorys(ctx, categoryFilter, categoryOpt)
}()
go func() {
defer wg.Done()
translationFilter := bson.M{
translationmod.FN.TranslationInfo.Status: true,
}
translationOpt := &options.FindOptions{}
translationOpt.SetLimit(1000).SetSort(bson.M{translationmod.FN.TranslationInfo.Rank: -1})
translationList, _ = translationmod.FindManyTranslationInfos(ctx, translationFilter, translationOpt)
}()
wg.Wait()
// 按 categoryId 建索引,避免嵌套遍历
translationByCategory := make(map[int64][]*translationmod.TranslationInfo, len(translationList))
for _, t := range translationList {
translationByCategory[t.TransCategoryId] = append(translationByCategory[t.TransCategoryId], t)
}
for _, categoryInfo := range translationCategoryList {
if categoryInfo.CategoryType == pubdef.TranslationTypeUrl {
if strings.HasPrefix(categoryInfo.UrlValue, "http://") || strings.HasPrefix(categoryInfo.UrlValue, "https://") {
continue
}
urlList = append(urlList, utils.JoinUrlPathAppendEnd(config.WebAddr, categoryInfo.UrlValue))
} else if categoryInfo.CategoryType == pubdef.TranslationTypeSub || categoryInfo.CategoryType == pubdef.TranslationTypeBottom {
for _, translationInfo := range translationByCategory[categoryInfo.Id] {
if translationInfo.IsFriendAds {
continue
} else if translationInfo.HtmlId > 0 {
title := strings.TrimSpace(translationInfo.Name)
if title != "" {
urlList = append(urlList, utils.JoinUrlPathKeepSlash(config.WebAddr, "/richtext?title="+url.QueryEscape(title)))
}
} else if len(translationInfo.Link) > 0 {
if !strings.HasPrefix(translationInfo.Link, "http://") && !strings.HasPrefix(translationInfo.Link, "https://") {
urlList = append(urlList, utils.JoinUrlPathAppendEnd(config.WebAddr, translationInfo.Link))
}
}
}
}
}
return urlList
}
+108
View File
@@ -0,0 +1,108 @@
package sitemap
import (
"testing"
"time"
"videoweb/pub/models/videocategorymod"
"videoweb/pub/models/videoinformationmod"
"go.mongodb.org/mongo-driver/bson"
)
func TestIsSitemapChannelPath(t *testing.T) {
if !isSitemapChannelPath("/drama", "/drama", "/drama-rank") {
t.Fatal("channel home should match")
}
if !isSitemapChannelPath("/drama/all", "/drama", "/drama-rank") {
t.Fatal("channel child path should match")
}
if !isSitemapChannelPath("/drama-rank", "/drama", "/drama-rank") {
t.Fatal("hyphen rank path should match")
}
if isSitemapChannelPath("/drama-rank", "/drama") {
t.Fatal("hyphen rank must not match prefix-only")
}
if isSitemapChannelPath("/video-rank", "/drama", "/drama-rank") {
t.Fatal("other channel rank should not match")
}
}
func TestBuildDetailURLs(t *testing.T) {
items := []idWithTime{
{Id: 53942, Slug: "sirozFs", LastMod: time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC)},
}
urls := buildDetailURLs("https://example.com/", "/video-detail", items, "weekly", 0.8)
if len(urls) != 1 {
t.Fatalf("buildDetailURLs() returned %d URLs, want 1", len(urls))
}
if want := "https://example.com/video-detail/sirozFs"; urls[0].Loc != want {
t.Fatalf("buildDetailURLs() loc = %q, want %q", urls[0].Loc, want)
}
fallback := buildDetailURLs("https://example.com/", "/video-detail", []idWithTime{{Id: 12}}, "weekly", 0.8)
if len(fallback) != 1 || fallback[0].Loc != "https://example.com/video-detail/12" {
t.Fatalf("empty slug should fall back to id, got %#v", fallback)
}
}
func TestBuildTagURLs(t *testing.T) {
slugURL := buildTagURLs("https://example.com/", "/video", []idWithTime{{Id: 9, Slug: "saduanfa"}}, "weekly", 0.6)
if len(slugURL) != 1 || slugURL[0].Loc != "https://example.com/video/t-saduanfa" {
t.Fatalf("slug tag loc = %#v", slugURL)
}
idURL := buildTagURLs("https://example.com/", "/drama", []idWithTime{{Id: 10142}}, "weekly", 0.6)
if len(idURL) != 1 || idURL[0].Loc != "https://example.com/drama/t-10142" {
t.Fatalf("id tag loc = %#v", idURL)
}
}
func TestBuildCategoryURLs(t *testing.T) {
slugURL := buildCategoryURLs("https://example.com/", "/video", []idWithTime{{Id: 37513, Slug: "hot-av"}}, "weekly", 0.6)
if len(slugURL) != 1 || slugURL[0].Loc != "https://example.com/video/c-hot-av" {
t.Fatalf("slug category loc = %#v", slugURL)
}
idURL := buildCategoryURLs("https://example.com/", "/drama", []idWithTime{{Id: 10001}}, "weekly", 0.6)
if len(idURL) != 1 || idURL[0].Loc != "https://example.com/drama/c-10001" {
t.Fatalf("id category loc = %#v", idURL)
}
}
func TestBuildPlayletSitemapURLs(t *testing.T) {
items := []idWithTime{{Id: 88, Slug: "chuanchengjiaofubei", LastMod: time.Date(2026, 9, 4, 0, 0, 0, 0, time.UTC)}}
detail := buildDetailURLs("https://example.com/", "/drama-detail", items, "weekly", 0.8)
if len(detail) != 1 || detail[0].Loc != "https://example.com/drama-detail/chuanchengjiaofubei" {
t.Fatalf("playlet detail loc = %#v", detail)
}
category := buildCategoryURLs("https://example.com/", "/drama", items, "weekly", 0.6)
if len(category) != 1 || category[0].Loc != "https://example.com/drama/c-chuanchengjiaofubei" {
t.Fatalf("playlet category loc = %#v", category)
}
idOnly := buildCategoryURLs("https://example.com/", "/video", []idWithTime{{Id: 37513}}, "weekly", 0.6)
if len(idOnly) != 1 || idOnly[0].Loc != "https://example.com/video/c-37513" {
t.Fatalf("category id fallback loc = %#v", idOnly)
}
tag := buildTagURLs("https://example.com/", "/drama", items, "weekly", 0.6)
if len(tag) != 1 || tag[0].Loc != "https://example.com/drama/t-chuanchengjiaofubei" {
t.Fatalf("playlet tag loc = %#v", tag)
}
}
func TestVideoCategoryContentFilter(t *testing.T) {
filter := videoCategoryContentFilter(&videocategorymod.VideoCategory{
Status: true,
Tags: []int64{11, 22},
})
tagsFilter, ok := filter[videoinformationmod.FN.VideoInformation.Tags].(bson.M)
if !ok {
t.Fatalf("tags filter = %#v, want bson.M", filter[videoinformationmod.FN.VideoInformation.Tags])
}
in, ok := tagsFilter["$in"].([]int64)
if !ok || len(in) != 2 || in[0] != 11 || in[1] != 22 {
t.Fatalf("tags $in = %#v, want [11 22]", tagsFilter["$in"])
}
emptyTags := videoCategoryContentFilter(&videocategorymod.VideoCategory{Status: true})
if _, exists := emptyTags[videoinformationmod.FN.VideoInformation.Tags]; exists {
t.Fatalf("empty tag category should not add tags filter: %#v", emptyTags)
}
}
+281
View File
@@ -0,0 +1,281 @@
package sitemap
import (
"context"
"fmt"
"net/url"
"strings"
"sync"
"time"
"videoweb/pub/common/utils"
"videoweb/pub/models/contenttagmod"
"videoweb/pub/models/playlethomeconfigmod"
"videoweb/pub/models/playletthememod"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
h5SortHot = "hot"
h5SortLatest = "latest"
h5TopicSort = "editorial"
h5RankPlaylet = "drama"
h5RankVideo = "video"
)
// getUrl 按 Next H5 路由生成 sitemap 分片:页面、短剧详情、视频详情。
func getUrl(sg *SitemapGenerator) {
webAddr := strings.TrimRight(strings.TrimSpace(sg.WebAddr), "/")
if webAddr == "" {
return
}
ctx := context.Background()
var (
wg sync.WaitGroup
videoEntries []videoEntry
videoTag map[int64]struct{}
playletItems []idWithTime
playletTagIDs map[int64]struct{}
themes []idWithTime
homeModules []idWithTime
)
wg.Add(4)
go func() {
defer wg.Done()
videoEntries, videoTag, _ = getVideoList(ctx)
}()
go func() {
defer wg.Done()
playletItems, _, playletTagIDs = getPlayletList(ctx)
}()
go func() {
defer wg.Done()
themes = getEnabledPlayletThemes(ctx)
}()
go func() {
defer wg.Done()
homeModules = getEnabledHomeModules(ctx)
}()
wg.Wait()
videoCatItems := getVideoCategoryList(ctx, videoTag, len(videoEntries) > 0)
videoTagItems := getContentTagList(ctx, videoTag, contenttagmod.ContentTagTypeVideo)
playletTagItems := getContentTagList(ctx, playletTagIDs, contenttagmod.ContentTagTypePlaylet)
now := time.Now()
pages := make([]URL, 0, 64)
seen := make(map[string]struct{})
add := func(dst *[]URL, loc, changeFreq string, priority float32, lastMod time.Time) {
loc = strings.TrimSpace(loc)
if loc == "" {
return
}
if _, ok := seen[loc]; ok {
return
}
seen[loc] = struct{}{}
if lastMod.IsZero() {
lastMod = now
}
*dst = append(*dst, URL{
Loc: loc,
LastMod: lastMod.Format(time.DateOnly),
ChangeFreq: changeFreq,
Priority: priority,
})
}
for _, spec := range h5StaticPages() {
add(&pages, h5Abs(webAddr, spec.path), spec.changeFreq, spec.priority, now)
}
for _, sort := range []string{h5SortHot, h5SortLatest} {
add(&pages, h5Abs(webAddr, "/short-drama", sort), "daily", 0.85, now)
add(&pages, h5Abs(webAddr, "/video", sort), "daily", 0.85, now)
for _, theme := range themes {
seg := detailSegment(theme.Slug, theme.Id)
if seg == "" || isReservedListSegment(seg) {
continue
}
add(&pages, h5Abs(webAddr, "/short-drama", sort, seg), "daily", 0.7, theme.LastMod)
}
for _, category := range videoCatItems {
seg := detailSegment(category.Slug, category.Id)
if seg == "" || isReservedListSegment(seg) {
continue
}
add(&pages, h5Abs(webAddr, "/video", sort, seg), "daily", 0.7, category.LastMod)
}
}
for _, tag := range playletTagItems {
seg := detailSegment(tag.Slug, tag.Id)
if seg == "" {
continue
}
add(&pages, h5Abs(webAddr, "/tag/playlet", seg), "weekly", 0.65, tag.LastMod)
}
for _, theme := range themes {
seg := detailSegment(theme.Slug, theme.Id)
if seg == "" {
continue
}
add(&pages, h5Abs(webAddr, "/topic/playlet", seg, h5TopicSort), "weekly", 0.7, theme.LastMod)
}
for _, tag := range videoTagItems {
seg := detailSegment(tag.Slug, tag.Id)
if seg == "" {
continue
}
add(&pages, h5Abs(webAddr, "/tag/video", seg), "weekly", 0.65, tag.LastMod)
}
for _, category := range videoCatItems {
seg := detailSegment(category.Slug, category.Id)
if seg == "" {
continue
}
add(&pages, h5Abs(webAddr, "/topic/video", seg, h5TopicSort), "weekly", 0.7, category.LastMod)
}
for _, module := range homeModules {
if module.Id <= 0 {
continue
}
add(&pages, h5Abs(webAddr, "/more", moreSegment(module.Slug, module.Id)), "daily", 0.65, module.LastMod)
}
if len(pages) > 0 {
_ = sg.writeSitemapFile("sitemap-pages.xml", pages)
}
playletURLs := make([]URL, 0, len(playletItems))
for _, item := range playletItems {
seg := detailSegment(item.Slug, item.Id)
if seg == "" || isReservedListSegment(seg) {
continue
}
add(&playletURLs, h5Abs(webAddr, "/short-drama", seg), "weekly", 0.8, item.LastMod)
}
if len(playletURLs) > 0 {
_ = sg.writeSitemapFiles("sitemap-playletdetail", playletURLs)
}
videoURLs := make([]URL, 0, len(videoEntries))
for _, item := range videoEntries {
seg := detailSegment(item.Slug, item.Id)
if seg == "" || isReservedListSegment(seg) {
continue
}
add(&videoURLs, h5Abs(webAddr, "/video", seg), "weekly", 0.8, item.LastMod)
}
if len(videoURLs) > 0 {
_ = sg.writeSitemapFiles("sitemap-videodetail", videoURLs)
}
}
type h5StaticPage struct {
path string
changeFreq string
priority float32
}
func h5StaticPages() []h5StaticPage {
return []h5StaticPage{
{path: "/", changeFreq: "daily", priority: 1},
{path: "/welfare", changeFreq: "weekly", priority: 0.6},
{path: "/nav", changeFreq: "weekly", priority: 0.7},
{path: "/way-home", changeFreq: "weekly", priority: 0.7},
{path: "/about", changeFreq: "monthly", priority: 0.4},
{path: "/terms", changeFreq: "monthly", priority: 0.3},
{path: "/privacy", changeFreq: "monthly", priority: 0.3},
{path: "/help", changeFreq: "monthly", priority: 0.3},
{path: "/ranking/" + h5RankPlaylet, changeFreq: "daily", priority: 0.8},
{path: "/ranking/" + h5RankVideo, changeFreq: "daily", priority: 0.8},
}
}
func isReservedListSegment(seg string) bool {
return seg == h5SortHot || seg == h5SortLatest || seg == h5RankPlaylet || seg == h5RankVideo
}
// moreSegment 首页更多页使用配置 slug。历史数据没有 slug 时使用 backfill 的字母标识。
func moreSegment(slug string, id int64) string {
if s := strings.TrimSpace(slug); s != "" {
return s
}
if id > 0 {
return fmt.Sprintf("home-config-%d", id)
}
return ""
}
// h5Abs 拼出与前端 encodeURIComponent 路径段一致的绝对地址。
func h5Abs(webAddr string, parts ...string) string {
segments := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.Trim(part, "/")
if part == "" {
continue
}
for _, seg := range strings.Split(part, "/") {
seg = strings.TrimSpace(seg)
if seg == "" {
continue
}
segments = append(segments, url.PathEscape(seg))
}
}
path := "/"
if len(segments) > 0 {
path = "/" + strings.Join(segments, "/")
}
return utils.JoinUrlPathKeepSlash(webAddr, path)
}
func getEnabledPlayletThemes(ctx context.Context) []idWithTime {
filter := bson.M{playletthememod.FN.PlayletTheme.Status: true}
pageSize := int64(1000)
items := make([]idWithTime, 0)
for i := int64(0); i < 100; i++ {
list, _ := playletthememod.FindManyPlayletTheme(ctx, filter, options.Find().
SetSort(bson.M{playletthememod.FN.PlayletTheme.Rank: -1}).
SetSkip(i*pageSize).
SetLimit(pageSize).
SetProjection(bson.M{
playletthememod.FN.PlayletTheme.Id: 1,
playletthememod.FN.PlayletTheme.Slug: 1,
playletthememod.FN.PlayletTheme.UpdatedAt: 1,
}))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
if len(list) < int(pageSize) {
break
}
}
return items
}
func getEnabledHomeModules(ctx context.Context) []idWithTime {
filter := bson.M{playlethomeconfigmod.FN.PlayletHomeConfig.Enabled: true}
list, _ := playlethomeconfigmod.FindManyPlayletHomeConfig(ctx, filter, options.Find().
SetProjection(bson.M{
playlethomeconfigmod.FN.PlayletHomeConfig.Id: 1,
playlethomeconfigmod.FN.PlayletHomeConfig.Slug: 1,
playlethomeconfigmod.FN.PlayletHomeConfig.UpdatedAt: 1,
}))
items := make([]idWithTime, 0, len(list))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
return items
}
+136
View File
@@ -0,0 +1,136 @@
package sitemap
import (
"context"
"videoweb/pub/models/contenttagmod"
"videoweb/pub/models/playletcategorymod"
"videoweb/pub/models/playletmod"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
// getPlayletList 拉取已上架短剧,并收集实际用到的分类/标签 ID。
func getPlayletList(ctx context.Context) ([]idWithTime, map[int64]struct{}, map[int64]struct{}) {
filter := bson.M{
playletmod.FN.Playlet.Status: playletmod.StatusUp,
}
pageSize := int64(1000)
items := make([]idWithTime, 0)
categoryIDs := make(map[int64]struct{})
tagIDs := make(map[int64]struct{})
for i := int64(0); i < 500; i++ {
skip := i * pageSize
list, _ := playletmod.FindManyPlaylet(ctx, filter, options.Find().
SetSort(bson.M{playletmod.FN.Playlet.Id: -1}).
SetSkip(skip).
SetLimit(pageSize).
SetProjection(bson.M{
playletmod.FN.Playlet.Id: 1,
playletmod.FN.Playlet.Slug: 1,
playletmod.FN.Playlet.UpdatedAt: 1,
playletmod.FN.Playlet.OnlineAt: 1,
playletmod.FN.Playlet.CategoryIds: 1,
playletmod.FN.Playlet.TagIds: 1,
}))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
lastMod := info.UpdatedAt
if lastMod.IsZero() {
lastMod = info.OnlineAt
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod})
for _, id := range info.CategoryIds {
if id > 0 {
categoryIDs[id] = struct{}{}
}
}
for _, id := range info.TagIds {
if id > 0 {
tagIDs[id] = struct{}{}
}
}
}
if len(list) < int(pageSize) {
break
}
}
return items, categoryIDs, tagIDs
}
// getPlayletCategoryList 只收录有上架短剧的分类,避免空列表页被抓取。
func getPlayletCategoryList(ctx context.Context, usedCategoryIDs map[int64]struct{}) []idWithTime {
if len(usedCategoryIDs) == 0 {
return nil
}
filter := bson.M{
playletcategorymod.FN.PlayletCategory.Status: true,
}
pageSize := int64(1000)
items := make([]idWithTime, 0)
for i := int64(0); i < 100; i++ {
skip := i * pageSize
list, _ := playletcategorymod.FindManyPlayletCategory(ctx, filter, options.Find().
SetSort(bson.M{playletcategorymod.FN.PlayletCategory.Rank: -1}).
SetSkip(skip).
SetLimit(pageSize).
SetProjection(bson.M{
playletcategorymod.FN.PlayletCategory.Id: 1,
playletcategorymod.FN.PlayletCategory.Slug: 1,
playletcategorymod.FN.PlayletCategory.UpdatedAt: 1,
}))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
if _, ok := usedCategoryIDs[info.Id]; !ok {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
if len(list) < int(pageSize) {
break
}
}
return items
}
// getContentTagList 只收录当前频道已绑定内容的统一标签。
func getContentTagList(ctx context.Context, usedTagIDs map[int64]struct{}, tagType contenttagmod.ContentTagType) []idWithTime {
if len(usedTagIDs) == 0 {
return nil
}
filter := bson.M{
contenttagmod.FN.ContentTag.Status: true,
contenttagmod.FN.ContentTag.Types: tagType,
}
pageSize := int64(1000)
items := make([]idWithTime, 0)
for i := int64(0); i < 100; i++ {
skip := i * pageSize
list, _ := contenttagmod.FindManyContentTag(ctx, filter, options.Find().
SetSort(bson.M{contenttagmod.FN.ContentTag.Rank: -1}).
SetSkip(skip).
SetLimit(pageSize).
SetProjection(bson.M{
contenttagmod.FN.ContentTag.Id: 1,
contenttagmod.FN.ContentTag.Slug: 1,
contenttagmod.FN.ContentTag.UpdatedAt: 1,
}))
for _, info := range list {
if info == nil || info.Id <= 0 {
continue
}
if _, ok := usedTagIDs[info.Id]; !ok {
continue
}
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
}
if len(list) < int(pageSize) {
break
}
}
return items
}
+245
View File
@@ -0,0 +1,245 @@
package sitemap
import (
"encoding/xml"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
const (
// 浏览器用 XSLT 把 loc 渲成可点击链接;搜索引擎仍按 XML 解析
stylesheetIndex = `<?xml-stylesheet type="text/xsl" href="/static/sitemapindex.xsl"?>` + "\n"
stylesheetURLSet = `<?xml-stylesheet type="text/xsl" href="/static/sitemap.xsl"?>` + "\n"
// defaultMaxURLsPerFile 单文件 URL 上限;Google 上限 50000,留余量避免体积过大。
defaultMaxURLsPerFile = 40000
)
// URL 单个URL条目
type URL struct {
Loc string `xml:"loc"`
LastMod string `xml:"lastmod"`
ChangeFreq string `xml:"changefreq"` // always/hourly/daily/weekly/monthly/yearly/never
Priority float32 `xml:"priority"` // 0.0 - 1.0
}
// Sitemap 站点地图文件
type Sitemap struct {
XMLName xml.Name `xml:"urlset"`
Xmlns string `xml:"xmlns,attr"`
URLs []URL `xml:"url"`
}
// SitemapIndex 站点地图索引
type SitemapIndex struct {
XMLName xml.Name `xml:"sitemapindex"`
Xmlns string `xml:"xmlns,attr"`
Sitemaps []SitemapIndexEntry `xml:"sitemap"`
}
// SitemapIndexEntry 站点地图索引条目
type SitemapIndexEntry struct {
Loc string `xml:"loc"`
LastMod string `xml:"lastmod"`
}
// SitemapGenerator 站点地图生成器
type SitemapGenerator struct {
MaxURLsPerFile int // 每个sitemap文件最大URL数量
OutputDir string // 分片输出目录(/sitemaps)
IndexFile string // 索引文件路径;空则写到 OutputDir/sitemap.xml
WebAddr string // 网站地址
SitemapFiles []string // 生成的sitemap文件列表
}
// NewSitemapGenerator 创建新的站点地图生成器
func NewSitemapGenerator(webAddr, outputDir string) *SitemapGenerator {
return &SitemapGenerator{
MaxURLsPerFile: defaultMaxURLsPerFile,
OutputDir: outputDir,
WebAddr: webAddr,
SitemapFiles: make([]string, 0),
}
}
func (sg *SitemapGenerator) maxURLsPerFile() int {
if sg.MaxURLsPerFile > 0 {
return sg.MaxURLsPerFile
}
return defaultMaxURLsPerFile
}
// Generate 生成站点地图
func (sg *SitemapGenerator) Generate() error {
// 清空之前的文件列表
sg.SitemapFiles = make([]string, 0)
// 创建输出目录
if err := os.MkdirAll(sg.OutputDir, 0755); err != nil {
return err
}
deprecatedFiles := []string{
"sitemap-createpostlist.xml",
"sitemap-postlist.xml",
"sitemap-posttag.xml",
"sitemap-createvideolist.xml",
"sitemap-tag.xml",
"sitemap-translation.xml",
"sitemap-actress.xml",
"sitemap-videocategory.xml",
"sitemap-playletcategory.xml",
"sitemap-videotag.xml",
"sitemap-playlettag.xml",
"sitemap-archives.xml",
"sitemap-videos.xml",
}
for _, filename := range deprecatedFiles {
if err := os.Remove(filepath.Join(sg.OutputDir, filename)); err != nil && !os.IsNotExist(err) {
return err
}
}
sg.cleanupSitemapShards("sitemap-archives", 0)
sg.cleanupSitemapShards("sitemap-videos", 0)
if sg.IndexFile != "" && filepath.Clean(sg.IndexFile) != filepath.Clean(filepath.Join(sg.OutputDir, "sitemap.xml")) {
if err := os.Remove(filepath.Join(sg.OutputDir, "sitemap.xml")); err != nil && !os.IsNotExist(err) {
return err
}
}
getUrl(sg)
// 生成sitemap索引文件
return sg.generateSitemapIndex()
}
// writeSitemapFile 写入sitemap文件
func (sg *SitemapGenerator) writeSitemapFile(filename string, urls []URL) error {
sitemap := Sitemap{
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
URLs: urls,
}
filePath := filepath.Join(sg.OutputDir, filename)
file, err := os.Create(filePath)
if err != nil {
return err
}
defer file.Close()
// 添加XML头
file.WriteString(xml.Header)
file.WriteString(stylesheetURLSet)
encoder := xml.NewEncoder(file)
encoder.Indent("", " ")
if err := encoder.Encode(sitemap); err != nil {
return err
}
// 记录生成的文件
sg.SitemapFiles = append(sg.SitemapFiles, filename)
return nil
}
// writeSitemapFiles 写入 sitemap 分片;超过上限时拆为 baseName-0.xml、baseName-1.xml ...
func (sg *SitemapGenerator) writeSitemapFiles(baseName string, urls []URL) error {
baseName = strings.TrimSuffix(strings.TrimSpace(baseName), ".xml")
if baseName == "" {
return nil
}
if len(urls) == 0 {
sg.cleanupSitemapShards(baseName, 0)
return nil
}
limit := sg.maxURLsPerFile()
shardCount := (len(urls) + limit - 1) / limit
if shardCount == 1 {
sg.cleanupSitemapShards(baseName, 1)
return sg.writeSitemapFile(baseName+".xml", urls)
}
sg.cleanupSitemapShards(baseName, shardCount)
for i := 0; i < shardCount; i++ {
start := i * limit
end := start + limit
if end > len(urls) {
end = len(urls)
}
filename := fmt.Sprintf("%s-%d.xml", baseName, i)
if err := sg.writeSitemapFile(filename, urls[start:end]); err != nil {
return err
}
}
return nil
}
// cleanupSitemapShards 清理旧分片:shardCount=1 保留 baseName.xml;>1 保留 baseName-0..N-1.xml。
func (sg *SitemapGenerator) cleanupSitemapShards(baseName string, shardCount int) {
entries, err := os.ReadDir(sg.OutputDir)
if err != nil {
return
}
prefix := baseName + "-"
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ".xml") {
idxText := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".xml")
idx, err := strconv.Atoi(idxText)
if err != nil {
continue
}
if shardCount <= 1 || idx >= shardCount {
_ = os.Remove(filepath.Join(sg.OutputDir, name))
}
continue
}
if shardCount > 1 && name == baseName+".xml" {
_ = os.Remove(filepath.Join(sg.OutputDir, name))
}
}
}
// generateSitemapIndex 生成sitemap索引文件
func (sg *SitemapGenerator) generateSitemapIndex() error {
index := SitemapIndex{
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
Sitemaps: make([]SitemapIndexEntry, 0, len(sg.SitemapFiles)),
}
now := time.Now().Format(time.DateOnly)
webAddr := strings.TrimRight(strings.TrimSpace(sg.WebAddr), "/")
for _, filename := range sg.SitemapFiles {
index.Sitemaps = append(index.Sitemaps, SitemapIndexEntry{
Loc: fmt.Sprintf("%s/sitemaps/%s", webAddr, filename),
LastMod: now,
})
}
filePath := sg.IndexFile
if strings.TrimSpace(filePath) == "" {
filePath = filepath.Join(sg.OutputDir, "sitemap.xml")
}
if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
return err
}
file, err := os.Create(filePath)
if err != nil {
return err
}
defer file.Close()
// 添加XML头
file.WriteString(xml.Header)
file.WriteString(stylesheetIndex)
encoder := xml.NewEncoder(file)
encoder.Indent("", " ")
return encoder.Encode(index)
}
+180
View File
@@ -0,0 +1,180 @@
package sitemap
import (
"bytes"
"encoding/xml"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"unicode/utf8"
"videoweb/pub/common/netutil"
"videoweb/pub/common/utils"
)
const (
videoSitemapShardPrefix = "sitemap-videos-"
videoSitemapDefaultThumb = "/static/images/og-default.jpg"
videoSitemapDefaultDesc = "高清成人视频在线观看。"
videoSitemapEmbedPathFmt = "/video/embed?id=%d"
)
type videoEntry struct {
Id int64
Slug string
Title string
LastMod time.Time
UnencryptedPreMp4 string
ContentLoc string
}
// prepareVideoSitemapEntries 为全部视频补齐 sitemap 字段;有 UnencryptedPreMp4 时额外设置 content_loc。
func prepareVideoSitemapEntries(entries []videoEntry, cdn string) []videoEntry {
cdn = strings.TrimSpace(cdn)
out := make([]videoEntry, 0, len(entries))
for _, e := range entries {
if e.Id <= 0 {
continue
}
if cdn != "" {
if preMp4 := strings.TrimSpace(e.UnencryptedPreMp4); preMp4 != "" {
if contentLoc := netutil.MergeURL(cdn, preMp4); contentLoc != "" {
e.ContentLoc = contentLoc
}
}
}
out = append(out, e)
}
return out
}
func (sg *SitemapGenerator) writeVideoSitemapFiles(entries []videoEntry) error {
if len(entries) == 0 {
sg.cleanupVideoSitemapShards(0)
return nil
}
limit := sg.maxURLsPerFile()
shardCount := (len(entries) + limit - 1) / limit
sg.cleanupVideoSitemapShards(shardCount)
thumb := strings.TrimRight(sg.WebAddr, "/") + videoSitemapDefaultThumb
for i := 0; i < len(entries); i += limit {
end := i + limit
if end > len(entries) {
end = len(entries)
}
name := fmt.Sprintf("%s%d.xml", videoSitemapShardPrefix, i/limit)
if err := sg.writeOneVideoSitemap(name, entries[i:end], thumb); err != nil {
return err
}
sg.SitemapFiles = append(sg.SitemapFiles, name)
}
return nil
}
func (sg *SitemapGenerator) cleanupVideoSitemapShards(shardCount int) {
entries, err := os.ReadDir(sg.OutputDir)
if err != nil {
return
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if !strings.HasPrefix(name, videoSitemapShardPrefix) || !strings.HasSuffix(name, ".xml") {
continue
}
idxText := strings.TrimSuffix(strings.TrimPrefix(name, videoSitemapShardPrefix), ".xml")
idx, err := strconv.Atoi(idxText)
if err != nil || shardCount <= 0 || idx >= shardCount {
_ = os.Remove(filepath.Join(sg.OutputDir, name))
}
}
}
func (sg *SitemapGenerator) writeOneVideoSitemap(filename string, entries []videoEntry, thumb string) error {
var b bytes.Buffer
b.WriteString(xml.Header)
b.WriteString(stylesheetURLSet)
b.WriteString(`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"` + "\n")
b.WriteString(` xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">` + "\n")
for _, e := range entries {
if e.Id <= 0 {
continue
}
loc := videoEntryLoc(sg.WebAddr, e)
if loc == "" {
continue
}
playerLoc := videoEntryPlayerLoc(sg.WebAddr, e.Id)
title := xmlEscapeVideoSitemap(videoEntryTitle(e))
desc := xmlEscapeVideoSitemap(videoSitemapDefaultDesc)
lastMod := videoEntryLastMod(e)
b.WriteString(" <url>\n")
fmt.Fprintf(&b, " <loc>%s</loc>\n", xmlEscapeVideoSitemap(loc))
fmt.Fprintf(&b, " <lastmod>%s</lastmod>\n", xmlEscapeVideoSitemap(lastMod))
b.WriteString(" <changefreq>weekly</changefreq>\n")
b.WriteString(" <priority>0.8</priority>\n")
b.WriteString(" <video:video>\n")
fmt.Fprintf(&b, " <video:thumbnail_loc>%s</video:thumbnail_loc>\n", xmlEscapeVideoSitemap(thumb))
fmt.Fprintf(&b, " <video:title>%s</video:title>\n", title)
fmt.Fprintf(&b, " <video:description>%s</video:description>\n", desc)
if contentLoc := strings.TrimSpace(e.ContentLoc); contentLoc != "" {
fmt.Fprintf(&b, " <video:content_loc>%s</video:content_loc>\n", xmlEscapeVideoSitemap(contentLoc))
}
fmt.Fprintf(&b, " <video:player_loc>%s</video:player_loc>\n", xmlEscapeVideoSitemap(playerLoc))
b.WriteString(" <video:family_friendly>no</video:family_friendly>\n")
b.WriteString(" </video:video>\n")
b.WriteString(" </url>\n")
}
b.WriteString("</urlset>\n")
path := filepath.Join(sg.OutputDir, filename)
return os.WriteFile(path, b.Bytes(), 0644)
}
func videoEntryLoc(webAddr string, e videoEntry) string {
seg := detailSegment(e.Slug, e.Id)
if seg == "" {
return ""
}
return utils.JoinUrlPathKeepSlash(webAddr, "/video-detail/"+seg)
}
func videoEntryPlayerLoc(webAddr string, id int64) string {
return utils.JoinUrlPathKeepSlash(webAddr, fmt.Sprintf(videoSitemapEmbedPathFmt, id))
}
func videoEntryTitle(e videoEntry) string {
if title := strings.TrimSpace(e.Title); title != "" {
return truncateRunesVideoSitemap(title, 100)
}
return fmt.Sprintf("视频 %d", e.Id)
}
func videoEntryLastMod(e videoEntry) string {
if e.LastMod.IsZero() {
return time.Now().Format(time.DateOnly)
}
return e.LastMod.Format(time.DateOnly)
}
func xmlEscapeVideoSitemap(s string) string {
var buf bytes.Buffer
_ = xml.EscapeText(&buf, []byte(s))
return buf.String()
}
func truncateRunesVideoSitemap(s string, max int) string {
if max <= 0 || utf8.RuneCountInString(s) <= max {
return s
}
runes := []rune(s)
return string(runes[:max])
}