Initial commit: web_hghl_server
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,312 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/pub/common/utils"
|
||||
"videoweb/pub/models/actress"
|
||||
"videoweb/pub/models/cfg"
|
||||
"videoweb/pub/models/translationcategorymod"
|
||||
"videoweb/pub/models/translationmod"
|
||||
"videoweb/pub/models/videocategorymod"
|
||||
"videoweb/pub/models/videoinformationmod"
|
||||
"videoweb/pub/pubdef"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// idWithTime 携带 ID、slug 和实际更新时间
|
||||
type idWithTime struct {
|
||||
Id int64
|
||||
Slug string
|
||||
LastMod time.Time
|
||||
}
|
||||
|
||||
func hasAnyID(ids []int64, used map[int64]struct{}) bool {
|
||||
for _, id := range ids {
|
||||
if _, ok := used[id]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isSitemapChannelPath(path, prefix string, extras ...string) bool {
|
||||
if path == prefix || strings.HasPrefix(path, prefix+"/") {
|
||||
return true
|
||||
}
|
||||
for _, extra := range extras {
|
||||
if path == extra {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func detailSegment(slug string, id int64) string {
|
||||
if s := strings.TrimSpace(slug); s != "" {
|
||||
return s
|
||||
}
|
||||
if id > 0 {
|
||||
return strconv.FormatInt(id, 10)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func buildChannelKeyURLs(webAddr, channel, keyPrefix string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
urls := make([]URL, 0, len(items))
|
||||
channel = strings.TrimRight(channel, "/")
|
||||
for _, item := range items {
|
||||
seg := detailSegment(item.Slug, item.Id)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
lastMod := item.LastMod.Format(time.DateOnly)
|
||||
if item.LastMod.IsZero() {
|
||||
lastMod = time.Now().Format(time.DateOnly)
|
||||
}
|
||||
urls = append(urls, URL{
|
||||
Loc: utils.JoinUrlPathKeepSlash(webAddr, channel+"/"+keyPrefix+seg),
|
||||
LastMod: lastMod,
|
||||
ChangeFreq: changeFreq,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
// buildCategoryURLs 分类页:/{channel}/c-{slug},无 slug 时回退 c-{id}。
|
||||
func buildCategoryURLs(webAddr, channel string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
return buildChannelKeyURLs(webAddr, channel, "c-", items, changeFreq, priority)
|
||||
}
|
||||
|
||||
// buildTagURLs 标签页:/{channel}/t-{slug},无 slug 时回退 t-{id}。
|
||||
func buildTagURLs(webAddr, channel string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
return buildChannelKeyURLs(webAddr, channel, "t-", items, changeFreq, priority)
|
||||
}
|
||||
|
||||
// buildDetailURLs 详情页:/{prefix}/{slug},无 slug 时回退数字 id。
|
||||
func buildDetailURLs(webAddr, prefix string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
urls := make([]URL, 0, len(items))
|
||||
for _, item := range items {
|
||||
seg := detailSegment(item.Slug, item.Id)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
lastMod := item.LastMod.Format(time.DateOnly)
|
||||
if item.LastMod.IsZero() {
|
||||
lastMod = time.Now().Format(time.DateOnly)
|
||||
}
|
||||
urls = append(urls, URL{
|
||||
Loc: utils.JoinUrlPathKeepSlash(webAddr, prefix+"/"+seg),
|
||||
LastMod: lastMod,
|
||||
ChangeFreq: changeFreq,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
// buildURLs 根据 ID+时间列表批量构建 URL 条目
|
||||
func buildURLs(webAddr string, pathFmt string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
urls := make([]URL, 0, len(items))
|
||||
for _, item := range items {
|
||||
lastMod := item.LastMod.Format(time.DateOnly)
|
||||
if item.LastMod.IsZero() {
|
||||
lastMod = time.Now().Format(time.DateOnly)
|
||||
}
|
||||
urls = append(urls, URL{
|
||||
Loc: utils.JoinUrlPathKeepSlash(webAddr, fmt.Sprintf(pathFmt, item.Id)),
|
||||
LastMod: lastMod,
|
||||
ChangeFreq: changeFreq,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
// filterIdsWithTime 返回 source 中存在于 validMap 的条目(含 slug)。
|
||||
func filterIdsWithTime(source map[int64]struct{}, validMap map[int64]idWithTime) []idWithTime {
|
||||
result := make([]idWithTime, 0, len(source))
|
||||
for id := range source {
|
||||
if item, ok := validMap[id]; ok {
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func getVideoCategoryList(ctx context.Context, usedTagIds map[int64]struct{}, hasVideos bool) []idWithTime {
|
||||
filter := bson.M{
|
||||
videocategorymod.FN.VideoCategory.Status: true,
|
||||
}
|
||||
pageSize := int64(1000)
|
||||
items := make([]idWithTime, 0)
|
||||
for i := int64(0); i < 500; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := videocategorymod.FindManyVideoCategorys(ctx, filter, options.Find().SetSort(bson.M{videocategorymod.FN.VideoCategory.Id: -1}).SetSkip(skip).SetLimit(pageSize))
|
||||
for _, info := range list {
|
||||
if len(info.Tags) == 0 {
|
||||
if !hasVideos {
|
||||
continue
|
||||
}
|
||||
} else if !hasAnyID(info.Tags, usedTagIds) {
|
||||
continue
|
||||
}
|
||||
if !videoCategoryHasContent(ctx, info) {
|
||||
continue
|
||||
}
|
||||
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// videoCategoryContentFilter 与 app QueryVideoMore 使用相同的视频筛选条件。
|
||||
func videoCategoryContentFilter(category *videocategorymod.VideoCategory) bson.M {
|
||||
filter := bson.M{
|
||||
videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK,
|
||||
}
|
||||
if category != nil && len(category.Tags) > 0 {
|
||||
filter[videoinformationmod.FN.VideoInformation.Tags] = bson.M{"$in": category.Tags}
|
||||
}
|
||||
return filter
|
||||
}
|
||||
|
||||
func videoCategoryHasContent(ctx context.Context, category *videocategorymod.VideoCategory) bool {
|
||||
if category == nil || !category.Status {
|
||||
return false
|
||||
}
|
||||
count, err := videoinformationmod.CountVideoInformations(ctx, videoCategoryContentFilter(category))
|
||||
return err == nil && count > 0
|
||||
}
|
||||
|
||||
func getActress(ctx context.Context) map[int64]idWithTime {
|
||||
filter := bson.M{
|
||||
actress.FN.Actress.Status: true,
|
||||
}
|
||||
pageSize := int64(1000)
|
||||
pageIds := make(map[int64]idWithTime)
|
||||
for i := int64(0); i < 100; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := actress.FindManyActresss(ctx, filter, options.Find().SetSkip(skip).SetLimit(pageSize).SetSort(bson.M{actress.FN.Actress.Id: -1}))
|
||||
for _, info := range list {
|
||||
pageIds[info.Id] = idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return pageIds
|
||||
}
|
||||
|
||||
func videoEntriesToIDWithTime(entries []videoEntry) []idWithTime {
|
||||
items := make([]idWithTime, len(entries))
|
||||
for i, e := range entries {
|
||||
items[i] = idWithTime{Id: e.Id, Slug: e.Slug, LastMod: e.LastMod}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func getVideoList(ctx context.Context) ([]videoEntry, map[int64]struct{}, map[int64]struct{}) {
|
||||
filter := bson.M{videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK}
|
||||
pageSize := int64(1000)
|
||||
items := make([]videoEntry, 0)
|
||||
tagIds := make(map[int64]struct{})
|
||||
actressIds := make(map[int64]struct{})
|
||||
for i := int64(0); i < 500; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := videoinformationmod.FindManyVideoInformations(ctx, filter, options.Find().SetSort(bson.M{videoinformationmod.FN.VideoInformation.Id: -1}).SetLimit(pageSize).SetSkip(skip))
|
||||
for _, info := range list {
|
||||
items = append(items, videoEntry{
|
||||
Id: info.Id,
|
||||
Slug: info.Slug,
|
||||
Title: info.Title,
|
||||
LastMod: info.UpdatedAt,
|
||||
UnencryptedPreMp4: info.Video.UnencryptedPreMp4,
|
||||
})
|
||||
for _, tagId := range info.Tags {
|
||||
tagIds[tagId] = struct{}{}
|
||||
}
|
||||
for _, actressId := range info.ActressIds {
|
||||
if actressId > 0 {
|
||||
actressIds[actressId] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return items, tagIds, actressIds
|
||||
}
|
||||
|
||||
func getTranslation(config cfg.SysCfg, ctx context.Context) []string {
|
||||
urlList := make([]string, 0)
|
||||
wg := &sync.WaitGroup{}
|
||||
wg.Add(2)
|
||||
var translationCategoryList translationcategorymod.TranslationCategoryList
|
||||
var translationList translationmod.TranslationInfoList
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
categoryFilter := bson.M{
|
||||
translationcategorymod.FN.TranslationCategory.Status: true,
|
||||
}
|
||||
categoryOpt := &options.FindOptions{}
|
||||
categoryOpt.SetLimit(1000).SetSort(bson.M{translationcategorymod.FN.TranslationCategory.Rank: -1})
|
||||
|
||||
translationCategoryList, _ = translationcategorymod.FindManyTranslationCategorys(ctx, categoryFilter, categoryOpt)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
translationFilter := bson.M{
|
||||
translationmod.FN.TranslationInfo.Status: true,
|
||||
}
|
||||
translationOpt := &options.FindOptions{}
|
||||
translationOpt.SetLimit(1000).SetSort(bson.M{translationmod.FN.TranslationInfo.Rank: -1})
|
||||
|
||||
translationList, _ = translationmod.FindManyTranslationInfos(ctx, translationFilter, translationOpt)
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// 按 categoryId 建索引,避免嵌套遍历
|
||||
translationByCategory := make(map[int64][]*translationmod.TranslationInfo, len(translationList))
|
||||
for _, t := range translationList {
|
||||
translationByCategory[t.TransCategoryId] = append(translationByCategory[t.TransCategoryId], t)
|
||||
}
|
||||
|
||||
for _, categoryInfo := range translationCategoryList {
|
||||
if categoryInfo.CategoryType == pubdef.TranslationTypeUrl {
|
||||
if strings.HasPrefix(categoryInfo.UrlValue, "http://") || strings.HasPrefix(categoryInfo.UrlValue, "https://") {
|
||||
continue
|
||||
}
|
||||
urlList = append(urlList, utils.JoinUrlPathAppendEnd(config.WebAddr, categoryInfo.UrlValue))
|
||||
} else if categoryInfo.CategoryType == pubdef.TranslationTypeSub || categoryInfo.CategoryType == pubdef.TranslationTypeBottom {
|
||||
for _, translationInfo := range translationByCategory[categoryInfo.Id] {
|
||||
if translationInfo.IsFriendAds {
|
||||
continue
|
||||
} else if translationInfo.HtmlId > 0 {
|
||||
title := strings.TrimSpace(translationInfo.Name)
|
||||
if title != "" {
|
||||
urlList = append(urlList, utils.JoinUrlPathKeepSlash(config.WebAddr, "/richtext?title="+url.QueryEscape(title)))
|
||||
}
|
||||
} else if len(translationInfo.Link) > 0 {
|
||||
if !strings.HasPrefix(translationInfo.Link, "http://") && !strings.HasPrefix(translationInfo.Link, "https://") {
|
||||
urlList = append(urlList, utils.JoinUrlPathAppendEnd(config.WebAddr, translationInfo.Link))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return urlList
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
"videoweb/pub/models/videocategorymod"
|
||||
"videoweb/pub/models/videoinformationmod"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
func TestIsSitemapChannelPath(t *testing.T) {
|
||||
if !isSitemapChannelPath("/drama", "/drama", "/drama-rank") {
|
||||
t.Fatal("channel home should match")
|
||||
}
|
||||
if !isSitemapChannelPath("/drama/all", "/drama", "/drama-rank") {
|
||||
t.Fatal("channel child path should match")
|
||||
}
|
||||
if !isSitemapChannelPath("/drama-rank", "/drama", "/drama-rank") {
|
||||
t.Fatal("hyphen rank path should match")
|
||||
}
|
||||
if isSitemapChannelPath("/drama-rank", "/drama") {
|
||||
t.Fatal("hyphen rank must not match prefix-only")
|
||||
}
|
||||
if isSitemapChannelPath("/video-rank", "/drama", "/drama-rank") {
|
||||
t.Fatal("other channel rank should not match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDetailURLs(t *testing.T) {
|
||||
items := []idWithTime{
|
||||
{Id: 53942, Slug: "sirozFs", LastMod: time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC)},
|
||||
}
|
||||
urls := buildDetailURLs("https://example.com/", "/video-detail", items, "weekly", 0.8)
|
||||
if len(urls) != 1 {
|
||||
t.Fatalf("buildDetailURLs() returned %d URLs, want 1", len(urls))
|
||||
}
|
||||
if want := "https://example.com/video-detail/sirozFs"; urls[0].Loc != want {
|
||||
t.Fatalf("buildDetailURLs() loc = %q, want %q", urls[0].Loc, want)
|
||||
}
|
||||
|
||||
fallback := buildDetailURLs("https://example.com/", "/video-detail", []idWithTime{{Id: 12}}, "weekly", 0.8)
|
||||
if len(fallback) != 1 || fallback[0].Loc != "https://example.com/video-detail/12" {
|
||||
t.Fatalf("empty slug should fall back to id, got %#v", fallback)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTagURLs(t *testing.T) {
|
||||
slugURL := buildTagURLs("https://example.com/", "/video", []idWithTime{{Id: 9, Slug: "saduanfa"}}, "weekly", 0.6)
|
||||
if len(slugURL) != 1 || slugURL[0].Loc != "https://example.com/video/t-saduanfa" {
|
||||
t.Fatalf("slug tag loc = %#v", slugURL)
|
||||
}
|
||||
idURL := buildTagURLs("https://example.com/", "/drama", []idWithTime{{Id: 10142}}, "weekly", 0.6)
|
||||
if len(idURL) != 1 || idURL[0].Loc != "https://example.com/drama/t-10142" {
|
||||
t.Fatalf("id tag loc = %#v", idURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCategoryURLs(t *testing.T) {
|
||||
slugURL := buildCategoryURLs("https://example.com/", "/video", []idWithTime{{Id: 37513, Slug: "hot-av"}}, "weekly", 0.6)
|
||||
if len(slugURL) != 1 || slugURL[0].Loc != "https://example.com/video/c-hot-av" {
|
||||
t.Fatalf("slug category loc = %#v", slugURL)
|
||||
}
|
||||
idURL := buildCategoryURLs("https://example.com/", "/drama", []idWithTime{{Id: 10001}}, "weekly", 0.6)
|
||||
if len(idURL) != 1 || idURL[0].Loc != "https://example.com/drama/c-10001" {
|
||||
t.Fatalf("id category loc = %#v", idURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPlayletSitemapURLs(t *testing.T) {
|
||||
items := []idWithTime{{Id: 88, Slug: "chuanchengjiaofubei", LastMod: time.Date(2026, 9, 4, 0, 0, 0, 0, time.UTC)}}
|
||||
detail := buildDetailURLs("https://example.com/", "/drama-detail", items, "weekly", 0.8)
|
||||
if len(detail) != 1 || detail[0].Loc != "https://example.com/drama-detail/chuanchengjiaofubei" {
|
||||
t.Fatalf("playlet detail loc = %#v", detail)
|
||||
}
|
||||
category := buildCategoryURLs("https://example.com/", "/drama", items, "weekly", 0.6)
|
||||
if len(category) != 1 || category[0].Loc != "https://example.com/drama/c-chuanchengjiaofubei" {
|
||||
t.Fatalf("playlet category loc = %#v", category)
|
||||
}
|
||||
idOnly := buildCategoryURLs("https://example.com/", "/video", []idWithTime{{Id: 37513}}, "weekly", 0.6)
|
||||
if len(idOnly) != 1 || idOnly[0].Loc != "https://example.com/video/c-37513" {
|
||||
t.Fatalf("category id fallback loc = %#v", idOnly)
|
||||
}
|
||||
tag := buildTagURLs("https://example.com/", "/drama", items, "weekly", 0.6)
|
||||
if len(tag) != 1 || tag[0].Loc != "https://example.com/drama/t-chuanchengjiaofubei" {
|
||||
t.Fatalf("playlet tag loc = %#v", tag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoCategoryContentFilter(t *testing.T) {
|
||||
filter := videoCategoryContentFilter(&videocategorymod.VideoCategory{
|
||||
Status: true,
|
||||
Tags: []int64{11, 22},
|
||||
})
|
||||
tagsFilter, ok := filter[videoinformationmod.FN.VideoInformation.Tags].(bson.M)
|
||||
if !ok {
|
||||
t.Fatalf("tags filter = %#v, want bson.M", filter[videoinformationmod.FN.VideoInformation.Tags])
|
||||
}
|
||||
in, ok := tagsFilter["$in"].([]int64)
|
||||
if !ok || len(in) != 2 || in[0] != 11 || in[1] != 22 {
|
||||
t.Fatalf("tags $in = %#v, want [11 22]", tagsFilter["$in"])
|
||||
}
|
||||
|
||||
emptyTags := videoCategoryContentFilter(&videocategorymod.VideoCategory{Status: true})
|
||||
if _, exists := emptyTags[videoinformationmod.FN.VideoInformation.Tags]; exists {
|
||||
t.Fatalf("empty tag category should not add tags filter: %#v", emptyTags)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"videoweb/pub/common/utils"
|
||||
"videoweb/pub/models/contenttagmod"
|
||||
"videoweb/pub/models/playlethomeconfigmod"
|
||||
"videoweb/pub/models/playletthememod"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
const (
|
||||
h5SortHot = "hot"
|
||||
h5SortLatest = "latest"
|
||||
h5TopicSort = "editorial"
|
||||
h5RankPlaylet = "drama"
|
||||
h5RankVideo = "video"
|
||||
)
|
||||
|
||||
// getUrl 按 Next H5 路由生成 sitemap 分片:页面、短剧详情、视频详情。
|
||||
func getUrl(sg *SitemapGenerator) {
|
||||
webAddr := strings.TrimRight(strings.TrimSpace(sg.WebAddr), "/")
|
||||
if webAddr == "" {
|
||||
return
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
|
||||
videoEntries []videoEntry
|
||||
videoTag map[int64]struct{}
|
||||
|
||||
playletItems []idWithTime
|
||||
playletTagIDs map[int64]struct{}
|
||||
|
||||
themes []idWithTime
|
||||
homeModules []idWithTime
|
||||
)
|
||||
|
||||
wg.Add(4)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
videoEntries, videoTag, _ = getVideoList(ctx)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
playletItems, _, playletTagIDs = getPlayletList(ctx)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
themes = getEnabledPlayletThemes(ctx)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
homeModules = getEnabledHomeModules(ctx)
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
videoCatItems := getVideoCategoryList(ctx, videoTag, len(videoEntries) > 0)
|
||||
videoTagItems := getContentTagList(ctx, videoTag, contenttagmod.ContentTagTypeVideo)
|
||||
playletTagItems := getContentTagList(ctx, playletTagIDs, contenttagmod.ContentTagTypePlaylet)
|
||||
|
||||
now := time.Now()
|
||||
pages := make([]URL, 0, 64)
|
||||
seen := make(map[string]struct{})
|
||||
add := func(dst *[]URL, loc, changeFreq string, priority float32, lastMod time.Time) {
|
||||
loc = strings.TrimSpace(loc)
|
||||
if loc == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[loc]; ok {
|
||||
return
|
||||
}
|
||||
seen[loc] = struct{}{}
|
||||
if lastMod.IsZero() {
|
||||
lastMod = now
|
||||
}
|
||||
*dst = append(*dst, URL{
|
||||
Loc: loc,
|
||||
LastMod: lastMod.Format(time.DateOnly),
|
||||
ChangeFreq: changeFreq,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
|
||||
for _, spec := range h5StaticPages() {
|
||||
add(&pages, h5Abs(webAddr, spec.path), spec.changeFreq, spec.priority, now)
|
||||
}
|
||||
for _, sort := range []string{h5SortHot, h5SortLatest} {
|
||||
add(&pages, h5Abs(webAddr, "/short-drama", sort), "daily", 0.85, now)
|
||||
add(&pages, h5Abs(webAddr, "/video", sort), "daily", 0.85, now)
|
||||
for _, theme := range themes {
|
||||
seg := detailSegment(theme.Slug, theme.Id)
|
||||
if seg == "" || isReservedListSegment(seg) {
|
||||
continue
|
||||
}
|
||||
add(&pages, h5Abs(webAddr, "/short-drama", sort, seg), "daily", 0.7, theme.LastMod)
|
||||
}
|
||||
for _, category := range videoCatItems {
|
||||
seg := detailSegment(category.Slug, category.Id)
|
||||
if seg == "" || isReservedListSegment(seg) {
|
||||
continue
|
||||
}
|
||||
add(&pages, h5Abs(webAddr, "/video", sort, seg), "daily", 0.7, category.LastMod)
|
||||
}
|
||||
}
|
||||
for _, tag := range playletTagItems {
|
||||
seg := detailSegment(tag.Slug, tag.Id)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
add(&pages, h5Abs(webAddr, "/tag/playlet", seg), "weekly", 0.65, tag.LastMod)
|
||||
}
|
||||
for _, theme := range themes {
|
||||
seg := detailSegment(theme.Slug, theme.Id)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
add(&pages, h5Abs(webAddr, "/topic/playlet", seg, h5TopicSort), "weekly", 0.7, theme.LastMod)
|
||||
}
|
||||
for _, tag := range videoTagItems {
|
||||
seg := detailSegment(tag.Slug, tag.Id)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
add(&pages, h5Abs(webAddr, "/tag/video", seg), "weekly", 0.65, tag.LastMod)
|
||||
}
|
||||
for _, category := range videoCatItems {
|
||||
seg := detailSegment(category.Slug, category.Id)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
add(&pages, h5Abs(webAddr, "/topic/video", seg, h5TopicSort), "weekly", 0.7, category.LastMod)
|
||||
}
|
||||
for _, module := range homeModules {
|
||||
if module.Id <= 0 {
|
||||
continue
|
||||
}
|
||||
add(&pages, h5Abs(webAddr, "/more", moreSegment(module.Slug, module.Id)), "daily", 0.65, module.LastMod)
|
||||
}
|
||||
if len(pages) > 0 {
|
||||
_ = sg.writeSitemapFile("sitemap-pages.xml", pages)
|
||||
}
|
||||
|
||||
playletURLs := make([]URL, 0, len(playletItems))
|
||||
for _, item := range playletItems {
|
||||
seg := detailSegment(item.Slug, item.Id)
|
||||
if seg == "" || isReservedListSegment(seg) {
|
||||
continue
|
||||
}
|
||||
add(&playletURLs, h5Abs(webAddr, "/short-drama", seg), "weekly", 0.8, item.LastMod)
|
||||
}
|
||||
if len(playletURLs) > 0 {
|
||||
_ = sg.writeSitemapFiles("sitemap-playletdetail", playletURLs)
|
||||
}
|
||||
|
||||
videoURLs := make([]URL, 0, len(videoEntries))
|
||||
for _, item := range videoEntries {
|
||||
seg := detailSegment(item.Slug, item.Id)
|
||||
if seg == "" || isReservedListSegment(seg) {
|
||||
continue
|
||||
}
|
||||
add(&videoURLs, h5Abs(webAddr, "/video", seg), "weekly", 0.8, item.LastMod)
|
||||
}
|
||||
if len(videoURLs) > 0 {
|
||||
_ = sg.writeSitemapFiles("sitemap-videodetail", videoURLs)
|
||||
}
|
||||
}
|
||||
|
||||
type h5StaticPage struct {
|
||||
path string
|
||||
changeFreq string
|
||||
priority float32
|
||||
}
|
||||
|
||||
func h5StaticPages() []h5StaticPage {
|
||||
return []h5StaticPage{
|
||||
{path: "/", changeFreq: "daily", priority: 1},
|
||||
{path: "/welfare", changeFreq: "weekly", priority: 0.6},
|
||||
{path: "/nav", changeFreq: "weekly", priority: 0.7},
|
||||
{path: "/way-home", changeFreq: "weekly", priority: 0.7},
|
||||
{path: "/about", changeFreq: "monthly", priority: 0.4},
|
||||
{path: "/terms", changeFreq: "monthly", priority: 0.3},
|
||||
{path: "/privacy", changeFreq: "monthly", priority: 0.3},
|
||||
{path: "/help", changeFreq: "monthly", priority: 0.3},
|
||||
{path: "/ranking/" + h5RankPlaylet, changeFreq: "daily", priority: 0.8},
|
||||
{path: "/ranking/" + h5RankVideo, changeFreq: "daily", priority: 0.8},
|
||||
}
|
||||
}
|
||||
|
||||
func isReservedListSegment(seg string) bool {
|
||||
return seg == h5SortHot || seg == h5SortLatest || seg == h5RankPlaylet || seg == h5RankVideo
|
||||
}
|
||||
|
||||
// moreSegment 首页更多页使用配置 slug。历史数据没有 slug 时使用 backfill 的字母标识。
|
||||
func moreSegment(slug string, id int64) string {
|
||||
if s := strings.TrimSpace(slug); s != "" {
|
||||
return s
|
||||
}
|
||||
if id > 0 {
|
||||
return fmt.Sprintf("home-config-%d", id)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// h5Abs 拼出与前端 encodeURIComponent 路径段一致的绝对地址。
|
||||
func h5Abs(webAddr string, parts ...string) string {
|
||||
segments := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.Trim(part, "/")
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
for _, seg := range strings.Split(part, "/") {
|
||||
seg = strings.TrimSpace(seg)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
segments = append(segments, url.PathEscape(seg))
|
||||
}
|
||||
}
|
||||
path := "/"
|
||||
if len(segments) > 0 {
|
||||
path = "/" + strings.Join(segments, "/")
|
||||
}
|
||||
return utils.JoinUrlPathKeepSlash(webAddr, path)
|
||||
}
|
||||
|
||||
func getEnabledPlayletThemes(ctx context.Context) []idWithTime {
|
||||
filter := bson.M{playletthememod.FN.PlayletTheme.Status: true}
|
||||
pageSize := int64(1000)
|
||||
items := make([]idWithTime, 0)
|
||||
for i := int64(0); i < 100; i++ {
|
||||
list, _ := playletthememod.FindManyPlayletTheme(ctx, filter, options.Find().
|
||||
SetSort(bson.M{playletthememod.FN.PlayletTheme.Rank: -1}).
|
||||
SetSkip(i*pageSize).
|
||||
SetLimit(pageSize).
|
||||
SetProjection(bson.M{
|
||||
playletthememod.FN.PlayletTheme.Id: 1,
|
||||
playletthememod.FN.PlayletTheme.Slug: 1,
|
||||
playletthememod.FN.PlayletTheme.UpdatedAt: 1,
|
||||
}))
|
||||
for _, info := range list {
|
||||
if info == nil || info.Id <= 0 {
|
||||
continue
|
||||
}
|
||||
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func getEnabledHomeModules(ctx context.Context) []idWithTime {
|
||||
filter := bson.M{playlethomeconfigmod.FN.PlayletHomeConfig.Enabled: true}
|
||||
list, _ := playlethomeconfigmod.FindManyPlayletHomeConfig(ctx, filter, options.Find().
|
||||
SetProjection(bson.M{
|
||||
playlethomeconfigmod.FN.PlayletHomeConfig.Id: 1,
|
||||
playlethomeconfigmod.FN.PlayletHomeConfig.Slug: 1,
|
||||
playlethomeconfigmod.FN.PlayletHomeConfig.UpdatedAt: 1,
|
||||
}))
|
||||
items := make([]idWithTime, 0, len(list))
|
||||
for _, info := range list {
|
||||
if info == nil || info.Id <= 0 {
|
||||
continue
|
||||
}
|
||||
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
||||
}
|
||||
return items
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"videoweb/pub/models/contenttagmod"
|
||||
"videoweb/pub/models/playletcategorymod"
|
||||
"videoweb/pub/models/playletmod"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// getPlayletList 拉取已上架短剧,并收集实际用到的分类/标签 ID。
|
||||
func getPlayletList(ctx context.Context) ([]idWithTime, map[int64]struct{}, map[int64]struct{}) {
|
||||
filter := bson.M{
|
||||
playletmod.FN.Playlet.Status: playletmod.StatusUp,
|
||||
}
|
||||
pageSize := int64(1000)
|
||||
items := make([]idWithTime, 0)
|
||||
categoryIDs := make(map[int64]struct{})
|
||||
tagIDs := make(map[int64]struct{})
|
||||
for i := int64(0); i < 500; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := playletmod.FindManyPlaylet(ctx, filter, options.Find().
|
||||
SetSort(bson.M{playletmod.FN.Playlet.Id: -1}).
|
||||
SetSkip(skip).
|
||||
SetLimit(pageSize).
|
||||
SetProjection(bson.M{
|
||||
playletmod.FN.Playlet.Id: 1,
|
||||
playletmod.FN.Playlet.Slug: 1,
|
||||
playletmod.FN.Playlet.UpdatedAt: 1,
|
||||
playletmod.FN.Playlet.OnlineAt: 1,
|
||||
playletmod.FN.Playlet.CategoryIds: 1,
|
||||
playletmod.FN.Playlet.TagIds: 1,
|
||||
}))
|
||||
for _, info := range list {
|
||||
if info == nil || info.Id <= 0 {
|
||||
continue
|
||||
}
|
||||
lastMod := info.UpdatedAt
|
||||
if lastMod.IsZero() {
|
||||
lastMod = info.OnlineAt
|
||||
}
|
||||
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod})
|
||||
for _, id := range info.CategoryIds {
|
||||
if id > 0 {
|
||||
categoryIDs[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, id := range info.TagIds {
|
||||
if id > 0 {
|
||||
tagIDs[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return items, categoryIDs, tagIDs
|
||||
}
|
||||
|
||||
// getPlayletCategoryList 只收录有上架短剧的分类,避免空列表页被抓取。
|
||||
func getPlayletCategoryList(ctx context.Context, usedCategoryIDs map[int64]struct{}) []idWithTime {
|
||||
if len(usedCategoryIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
filter := bson.M{
|
||||
playletcategorymod.FN.PlayletCategory.Status: true,
|
||||
}
|
||||
pageSize := int64(1000)
|
||||
items := make([]idWithTime, 0)
|
||||
for i := int64(0); i < 100; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := playletcategorymod.FindManyPlayletCategory(ctx, filter, options.Find().
|
||||
SetSort(bson.M{playletcategorymod.FN.PlayletCategory.Rank: -1}).
|
||||
SetSkip(skip).
|
||||
SetLimit(pageSize).
|
||||
SetProjection(bson.M{
|
||||
playletcategorymod.FN.PlayletCategory.Id: 1,
|
||||
playletcategorymod.FN.PlayletCategory.Slug: 1,
|
||||
playletcategorymod.FN.PlayletCategory.UpdatedAt: 1,
|
||||
}))
|
||||
for _, info := range list {
|
||||
if info == nil || info.Id <= 0 {
|
||||
continue
|
||||
}
|
||||
if _, ok := usedCategoryIDs[info.Id]; !ok {
|
||||
continue
|
||||
}
|
||||
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// getContentTagList 只收录当前频道已绑定内容的统一标签。
|
||||
func getContentTagList(ctx context.Context, usedTagIDs map[int64]struct{}, tagType contenttagmod.ContentTagType) []idWithTime {
|
||||
if len(usedTagIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
filter := bson.M{
|
||||
contenttagmod.FN.ContentTag.Status: true,
|
||||
contenttagmod.FN.ContentTag.Types: tagType,
|
||||
}
|
||||
pageSize := int64(1000)
|
||||
items := make([]idWithTime, 0)
|
||||
for i := int64(0); i < 100; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := contenttagmod.FindManyContentTag(ctx, filter, options.Find().
|
||||
SetSort(bson.M{contenttagmod.FN.ContentTag.Rank: -1}).
|
||||
SetSkip(skip).
|
||||
SetLimit(pageSize).
|
||||
SetProjection(bson.M{
|
||||
contenttagmod.FN.ContentTag.Id: 1,
|
||||
contenttagmod.FN.ContentTag.Slug: 1,
|
||||
contenttagmod.FN.ContentTag.UpdatedAt: 1,
|
||||
}))
|
||||
for _, info := range list {
|
||||
if info == nil || info.Id <= 0 {
|
||||
continue
|
||||
}
|
||||
if _, ok := usedTagIDs[info.Id]; !ok {
|
||||
continue
|
||||
}
|
||||
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// 浏览器用 XSLT 把 loc 渲成可点击链接;搜索引擎仍按 XML 解析
|
||||
stylesheetIndex = `<?xml-stylesheet type="text/xsl" href="/static/sitemapindex.xsl"?>` + "\n"
|
||||
stylesheetURLSet = `<?xml-stylesheet type="text/xsl" href="/static/sitemap.xsl"?>` + "\n"
|
||||
|
||||
// defaultMaxURLsPerFile 单文件 URL 上限;Google 上限 50000,留余量避免体积过大。
|
||||
defaultMaxURLsPerFile = 40000
|
||||
)
|
||||
|
||||
// URL 单个URL条目
|
||||
type URL struct {
|
||||
Loc string `xml:"loc"`
|
||||
LastMod string `xml:"lastmod"`
|
||||
ChangeFreq string `xml:"changefreq"` // always/hourly/daily/weekly/monthly/yearly/never
|
||||
Priority float32 `xml:"priority"` // 0.0 - 1.0
|
||||
}
|
||||
|
||||
// Sitemap 站点地图文件
|
||||
type Sitemap struct {
|
||||
XMLName xml.Name `xml:"urlset"`
|
||||
Xmlns string `xml:"xmlns,attr"`
|
||||
URLs []URL `xml:"url"`
|
||||
}
|
||||
|
||||
// SitemapIndex 站点地图索引
|
||||
type SitemapIndex struct {
|
||||
XMLName xml.Name `xml:"sitemapindex"`
|
||||
Xmlns string `xml:"xmlns,attr"`
|
||||
Sitemaps []SitemapIndexEntry `xml:"sitemap"`
|
||||
}
|
||||
|
||||
// SitemapIndexEntry 站点地图索引条目
|
||||
type SitemapIndexEntry struct {
|
||||
Loc string `xml:"loc"`
|
||||
LastMod string `xml:"lastmod"`
|
||||
}
|
||||
|
||||
// SitemapGenerator 站点地图生成器
|
||||
type SitemapGenerator struct {
|
||||
MaxURLsPerFile int // 每个sitemap文件最大URL数量
|
||||
OutputDir string // 分片输出目录(/sitemaps)
|
||||
IndexFile string // 索引文件路径;空则写到 OutputDir/sitemap.xml
|
||||
WebAddr string // 网站地址
|
||||
SitemapFiles []string // 生成的sitemap文件列表
|
||||
}
|
||||
|
||||
// NewSitemapGenerator 创建新的站点地图生成器
|
||||
func NewSitemapGenerator(webAddr, outputDir string) *SitemapGenerator {
|
||||
return &SitemapGenerator{
|
||||
MaxURLsPerFile: defaultMaxURLsPerFile,
|
||||
OutputDir: outputDir,
|
||||
WebAddr: webAddr,
|
||||
SitemapFiles: make([]string, 0),
|
||||
}
|
||||
}
|
||||
|
||||
func (sg *SitemapGenerator) maxURLsPerFile() int {
|
||||
if sg.MaxURLsPerFile > 0 {
|
||||
return sg.MaxURLsPerFile
|
||||
}
|
||||
return defaultMaxURLsPerFile
|
||||
}
|
||||
|
||||
// Generate 生成站点地图
|
||||
func (sg *SitemapGenerator) Generate() error {
|
||||
// 清空之前的文件列表
|
||||
sg.SitemapFiles = make([]string, 0)
|
||||
|
||||
// 创建输出目录
|
||||
if err := os.MkdirAll(sg.OutputDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
deprecatedFiles := []string{
|
||||
"sitemap-createpostlist.xml",
|
||||
"sitemap-postlist.xml",
|
||||
"sitemap-posttag.xml",
|
||||
"sitemap-createvideolist.xml",
|
||||
"sitemap-tag.xml",
|
||||
"sitemap-translation.xml",
|
||||
"sitemap-actress.xml",
|
||||
"sitemap-videocategory.xml",
|
||||
"sitemap-playletcategory.xml",
|
||||
"sitemap-videotag.xml",
|
||||
"sitemap-playlettag.xml",
|
||||
"sitemap-archives.xml",
|
||||
"sitemap-videos.xml",
|
||||
}
|
||||
for _, filename := range deprecatedFiles {
|
||||
if err := os.Remove(filepath.Join(sg.OutputDir, filename)); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
sg.cleanupSitemapShards("sitemap-archives", 0)
|
||||
sg.cleanupSitemapShards("sitemap-videos", 0)
|
||||
if sg.IndexFile != "" && filepath.Clean(sg.IndexFile) != filepath.Clean(filepath.Join(sg.OutputDir, "sitemap.xml")) {
|
||||
if err := os.Remove(filepath.Join(sg.OutputDir, "sitemap.xml")); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
getUrl(sg)
|
||||
|
||||
// 生成sitemap索引文件
|
||||
return sg.generateSitemapIndex()
|
||||
}
|
||||
|
||||
// writeSitemapFile 写入sitemap文件
|
||||
func (sg *SitemapGenerator) writeSitemapFile(filename string, urls []URL) error {
|
||||
sitemap := Sitemap{
|
||||
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||||
URLs: urls,
|
||||
}
|
||||
|
||||
filePath := filepath.Join(sg.OutputDir, filename)
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// 添加XML头
|
||||
file.WriteString(xml.Header)
|
||||
file.WriteString(stylesheetURLSet)
|
||||
encoder := xml.NewEncoder(file)
|
||||
encoder.Indent("", " ")
|
||||
|
||||
if err := encoder.Encode(sitemap); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 记录生成的文件
|
||||
sg.SitemapFiles = append(sg.SitemapFiles, filename)
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeSitemapFiles 写入 sitemap 分片;超过上限时拆为 baseName-0.xml、baseName-1.xml ...
|
||||
func (sg *SitemapGenerator) writeSitemapFiles(baseName string, urls []URL) error {
|
||||
baseName = strings.TrimSuffix(strings.TrimSpace(baseName), ".xml")
|
||||
if baseName == "" {
|
||||
return nil
|
||||
}
|
||||
if len(urls) == 0 {
|
||||
sg.cleanupSitemapShards(baseName, 0)
|
||||
return nil
|
||||
}
|
||||
|
||||
limit := sg.maxURLsPerFile()
|
||||
shardCount := (len(urls) + limit - 1) / limit
|
||||
if shardCount == 1 {
|
||||
sg.cleanupSitemapShards(baseName, 1)
|
||||
return sg.writeSitemapFile(baseName+".xml", urls)
|
||||
}
|
||||
|
||||
sg.cleanupSitemapShards(baseName, shardCount)
|
||||
for i := 0; i < shardCount; i++ {
|
||||
start := i * limit
|
||||
end := start + limit
|
||||
if end > len(urls) {
|
||||
end = len(urls)
|
||||
}
|
||||
filename := fmt.Sprintf("%s-%d.xml", baseName, i)
|
||||
if err := sg.writeSitemapFile(filename, urls[start:end]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupSitemapShards 清理旧分片:shardCount=1 保留 baseName.xml;>1 保留 baseName-0..N-1.xml。
|
||||
func (sg *SitemapGenerator) cleanupSitemapShards(baseName string, shardCount int) {
|
||||
entries, err := os.ReadDir(sg.OutputDir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
prefix := baseName + "-"
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ".xml") {
|
||||
idxText := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".xml")
|
||||
idx, err := strconv.Atoi(idxText)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if shardCount <= 1 || idx >= shardCount {
|
||||
_ = os.Remove(filepath.Join(sg.OutputDir, name))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if shardCount > 1 && name == baseName+".xml" {
|
||||
_ = os.Remove(filepath.Join(sg.OutputDir, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// generateSitemapIndex 生成sitemap索引文件
|
||||
func (sg *SitemapGenerator) generateSitemapIndex() error {
|
||||
index := SitemapIndex{
|
||||
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||||
Sitemaps: make([]SitemapIndexEntry, 0, len(sg.SitemapFiles)),
|
||||
}
|
||||
|
||||
now := time.Now().Format(time.DateOnly)
|
||||
webAddr := strings.TrimRight(strings.TrimSpace(sg.WebAddr), "/")
|
||||
for _, filename := range sg.SitemapFiles {
|
||||
index.Sitemaps = append(index.Sitemaps, SitemapIndexEntry{
|
||||
Loc: fmt.Sprintf("%s/sitemaps/%s", webAddr, filename),
|
||||
LastMod: now,
|
||||
})
|
||||
}
|
||||
|
||||
filePath := sg.IndexFile
|
||||
if strings.TrimSpace(filePath) == "" {
|
||||
filePath = filepath.Join(sg.OutputDir, "sitemap.xml")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// 添加XML头
|
||||
file.WriteString(xml.Header)
|
||||
file.WriteString(stylesheetIndex)
|
||||
encoder := xml.NewEncoder(file)
|
||||
encoder.Indent("", " ")
|
||||
|
||||
return encoder.Encode(index)
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"videoweb/pub/common/netutil"
|
||||
"videoweb/pub/common/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
videoSitemapShardPrefix = "sitemap-videos-"
|
||||
videoSitemapDefaultThumb = "/static/images/og-default.jpg"
|
||||
videoSitemapDefaultDesc = "高清成人视频在线观看。"
|
||||
videoSitemapEmbedPathFmt = "/video/embed?id=%d"
|
||||
)
|
||||
|
||||
type videoEntry struct {
|
||||
Id int64
|
||||
Slug string
|
||||
Title string
|
||||
LastMod time.Time
|
||||
UnencryptedPreMp4 string
|
||||
ContentLoc string
|
||||
}
|
||||
|
||||
// prepareVideoSitemapEntries 为全部视频补齐 sitemap 字段;有 UnencryptedPreMp4 时额外设置 content_loc。
|
||||
func prepareVideoSitemapEntries(entries []videoEntry, cdn string) []videoEntry {
|
||||
cdn = strings.TrimSpace(cdn)
|
||||
out := make([]videoEntry, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.Id <= 0 {
|
||||
continue
|
||||
}
|
||||
if cdn != "" {
|
||||
if preMp4 := strings.TrimSpace(e.UnencryptedPreMp4); preMp4 != "" {
|
||||
if contentLoc := netutil.MergeURL(cdn, preMp4); contentLoc != "" {
|
||||
e.ContentLoc = contentLoc
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (sg *SitemapGenerator) writeVideoSitemapFiles(entries []videoEntry) error {
|
||||
if len(entries) == 0 {
|
||||
sg.cleanupVideoSitemapShards(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
limit := sg.maxURLsPerFile()
|
||||
shardCount := (len(entries) + limit - 1) / limit
|
||||
sg.cleanupVideoSitemapShards(shardCount)
|
||||
|
||||
thumb := strings.TrimRight(sg.WebAddr, "/") + videoSitemapDefaultThumb
|
||||
for i := 0; i < len(entries); i += limit {
|
||||
end := i + limit
|
||||
if end > len(entries) {
|
||||
end = len(entries)
|
||||
}
|
||||
name := fmt.Sprintf("%s%d.xml", videoSitemapShardPrefix, i/limit)
|
||||
if err := sg.writeOneVideoSitemap(name, entries[i:end], thumb); err != nil {
|
||||
return err
|
||||
}
|
||||
sg.SitemapFiles = append(sg.SitemapFiles, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sg *SitemapGenerator) cleanupVideoSitemapShards(shardCount int) {
|
||||
entries, err := os.ReadDir(sg.OutputDir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
if !strings.HasPrefix(name, videoSitemapShardPrefix) || !strings.HasSuffix(name, ".xml") {
|
||||
continue
|
||||
}
|
||||
idxText := strings.TrimSuffix(strings.TrimPrefix(name, videoSitemapShardPrefix), ".xml")
|
||||
idx, err := strconv.Atoi(idxText)
|
||||
if err != nil || shardCount <= 0 || idx >= shardCount {
|
||||
_ = os.Remove(filepath.Join(sg.OutputDir, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sg *SitemapGenerator) writeOneVideoSitemap(filename string, entries []videoEntry, thumb string) error {
|
||||
var b bytes.Buffer
|
||||
b.WriteString(xml.Header)
|
||||
b.WriteString(stylesheetURLSet)
|
||||
b.WriteString(`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"` + "\n")
|
||||
b.WriteString(` xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">` + "\n")
|
||||
for _, e := range entries {
|
||||
if e.Id <= 0 {
|
||||
continue
|
||||
}
|
||||
loc := videoEntryLoc(sg.WebAddr, e)
|
||||
if loc == "" {
|
||||
continue
|
||||
}
|
||||
playerLoc := videoEntryPlayerLoc(sg.WebAddr, e.Id)
|
||||
title := xmlEscapeVideoSitemap(videoEntryTitle(e))
|
||||
desc := xmlEscapeVideoSitemap(videoSitemapDefaultDesc)
|
||||
lastMod := videoEntryLastMod(e)
|
||||
|
||||
b.WriteString(" <url>\n")
|
||||
fmt.Fprintf(&b, " <loc>%s</loc>\n", xmlEscapeVideoSitemap(loc))
|
||||
fmt.Fprintf(&b, " <lastmod>%s</lastmod>\n", xmlEscapeVideoSitemap(lastMod))
|
||||
b.WriteString(" <changefreq>weekly</changefreq>\n")
|
||||
b.WriteString(" <priority>0.8</priority>\n")
|
||||
b.WriteString(" <video:video>\n")
|
||||
fmt.Fprintf(&b, " <video:thumbnail_loc>%s</video:thumbnail_loc>\n", xmlEscapeVideoSitemap(thumb))
|
||||
fmt.Fprintf(&b, " <video:title>%s</video:title>\n", title)
|
||||
fmt.Fprintf(&b, " <video:description>%s</video:description>\n", desc)
|
||||
if contentLoc := strings.TrimSpace(e.ContentLoc); contentLoc != "" {
|
||||
fmt.Fprintf(&b, " <video:content_loc>%s</video:content_loc>\n", xmlEscapeVideoSitemap(contentLoc))
|
||||
}
|
||||
fmt.Fprintf(&b, " <video:player_loc>%s</video:player_loc>\n", xmlEscapeVideoSitemap(playerLoc))
|
||||
b.WriteString(" <video:family_friendly>no</video:family_friendly>\n")
|
||||
b.WriteString(" </video:video>\n")
|
||||
b.WriteString(" </url>\n")
|
||||
}
|
||||
b.WriteString("</urlset>\n")
|
||||
|
||||
path := filepath.Join(sg.OutputDir, filename)
|
||||
return os.WriteFile(path, b.Bytes(), 0644)
|
||||
}
|
||||
|
||||
func videoEntryLoc(webAddr string, e videoEntry) string {
|
||||
seg := detailSegment(e.Slug, e.Id)
|
||||
if seg == "" {
|
||||
return ""
|
||||
}
|
||||
return utils.JoinUrlPathKeepSlash(webAddr, "/video-detail/"+seg)
|
||||
}
|
||||
|
||||
func videoEntryPlayerLoc(webAddr string, id int64) string {
|
||||
return utils.JoinUrlPathKeepSlash(webAddr, fmt.Sprintf(videoSitemapEmbedPathFmt, id))
|
||||
}
|
||||
|
||||
func videoEntryTitle(e videoEntry) string {
|
||||
if title := strings.TrimSpace(e.Title); title != "" {
|
||||
return truncateRunesVideoSitemap(title, 100)
|
||||
}
|
||||
return fmt.Sprintf("视频 %d", e.Id)
|
||||
}
|
||||
|
||||
func videoEntryLastMod(e videoEntry) string {
|
||||
if e.LastMod.IsZero() {
|
||||
return time.Now().Format(time.DateOnly)
|
||||
}
|
||||
return e.LastMod.Format(time.DateOnly)
|
||||
}
|
||||
|
||||
func xmlEscapeVideoSitemap(s string) string {
|
||||
var buf bytes.Buffer
|
||||
_ = xml.EscapeText(&buf, []byte(s))
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func truncateRunesVideoSitemap(s string, max int) string {
|
||||
if max <= 0 || utf8.RuneCountInString(s) <= max {
|
||||
return s
|
||||
}
|
||||
runes := []rune(s)
|
||||
return string(runes[:max])
|
||||
}
|
||||
Reference in New Issue
Block a user