313 lines
9.7 KiB
Go
313 lines
9.7 KiB
Go
package sitemap
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
"videoweb/pub/common/utils"
|
|
"videoweb/pub/models/actress"
|
|
"videoweb/pub/models/cfg"
|
|
"videoweb/pub/models/translationcategorymod"
|
|
"videoweb/pub/models/translationmod"
|
|
"videoweb/pub/models/videocategorymod"
|
|
"videoweb/pub/models/videoinformationmod"
|
|
"videoweb/pub/pubdef"
|
|
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
// idWithTime 携带 ID、slug 和实际更新时间
|
|
type idWithTime struct {
|
|
Id int64
|
|
Slug string
|
|
LastMod time.Time
|
|
}
|
|
|
|
func hasAnyID(ids []int64, used map[int64]struct{}) bool {
|
|
for _, id := range ids {
|
|
if _, ok := used[id]; ok {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isSitemapChannelPath(path, prefix string, extras ...string) bool {
|
|
if path == prefix || strings.HasPrefix(path, prefix+"/") {
|
|
return true
|
|
}
|
|
for _, extra := range extras {
|
|
if path == extra {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func detailSegment(slug string, id int64) string {
|
|
if s := strings.TrimSpace(slug); s != "" {
|
|
return s
|
|
}
|
|
if id > 0 {
|
|
return strconv.FormatInt(id, 10)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func buildChannelKeyURLs(webAddr, channel, keyPrefix string, items []idWithTime, changeFreq string, priority float32) []URL {
|
|
urls := make([]URL, 0, len(items))
|
|
channel = strings.TrimRight(channel, "/")
|
|
for _, item := range items {
|
|
seg := detailSegment(item.Slug, item.Id)
|
|
if seg == "" {
|
|
continue
|
|
}
|
|
lastMod := item.LastMod.Format(time.DateOnly)
|
|
if item.LastMod.IsZero() {
|
|
lastMod = time.Now().Format(time.DateOnly)
|
|
}
|
|
urls = append(urls, URL{
|
|
Loc: utils.JoinUrlPathKeepSlash(webAddr, channel+"/"+keyPrefix+seg),
|
|
LastMod: lastMod,
|
|
ChangeFreq: changeFreq,
|
|
Priority: priority,
|
|
})
|
|
}
|
|
return urls
|
|
}
|
|
|
|
// buildCategoryURLs 分类页：/{channel}/c-{slug}，无 slug 时回退 c-{id}。
|
|
func buildCategoryURLs(webAddr, channel string, items []idWithTime, changeFreq string, priority float32) []URL {
|
|
return buildChannelKeyURLs(webAddr, channel, "c-", items, changeFreq, priority)
|
|
}
|
|
|
|
// buildTagURLs 标签页：/{channel}/t-{slug}，无 slug 时回退 t-{id}。
|
|
func buildTagURLs(webAddr, channel string, items []idWithTime, changeFreq string, priority float32) []URL {
|
|
return buildChannelKeyURLs(webAddr, channel, "t-", items, changeFreq, priority)
|
|
}
|
|
|
|
// buildDetailURLs 详情页：/{prefix}/{slug}，无 slug 时回退数字 id。
|
|
func buildDetailURLs(webAddr, prefix string, items []idWithTime, changeFreq string, priority float32) []URL {
|
|
urls := make([]URL, 0, len(items))
|
|
for _, item := range items {
|
|
seg := detailSegment(item.Slug, item.Id)
|
|
if seg == "" {
|
|
continue
|
|
}
|
|
lastMod := item.LastMod.Format(time.DateOnly)
|
|
if item.LastMod.IsZero() {
|
|
lastMod = time.Now().Format(time.DateOnly)
|
|
}
|
|
urls = append(urls, URL{
|
|
Loc: utils.JoinUrlPathKeepSlash(webAddr, prefix+"/"+seg),
|
|
LastMod: lastMod,
|
|
ChangeFreq: changeFreq,
|
|
Priority: priority,
|
|
})
|
|
}
|
|
return urls
|
|
}
|
|
|
|
// buildURLs 根据 ID+时间列表批量构建 URL 条目
|
|
func buildURLs(webAddr string, pathFmt string, items []idWithTime, changeFreq string, priority float32) []URL {
|
|
urls := make([]URL, 0, len(items))
|
|
for _, item := range items {
|
|
lastMod := item.LastMod.Format(time.DateOnly)
|
|
if item.LastMod.IsZero() {
|
|
lastMod = time.Now().Format(time.DateOnly)
|
|
}
|
|
urls = append(urls, URL{
|
|
Loc: utils.JoinUrlPathKeepSlash(webAddr, fmt.Sprintf(pathFmt, item.Id)),
|
|
LastMod: lastMod,
|
|
ChangeFreq: changeFreq,
|
|
Priority: priority,
|
|
})
|
|
}
|
|
return urls
|
|
}
|
|
|
|
// filterIdsWithTime 返回 source 中存在于 validMap 的条目（含 slug）。
|
|
func filterIdsWithTime(source map[int64]struct{}, validMap map[int64]idWithTime) []idWithTime {
|
|
result := make([]idWithTime, 0, len(source))
|
|
for id := range source {
|
|
if item, ok := validMap[id]; ok {
|
|
result = append(result, item)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func getVideoCategoryList(ctx context.Context, usedTagIds map[int64]struct{}, hasVideos bool) []idWithTime {
|
|
filter := bson.M{
|
|
videocategorymod.FN.VideoCategory.Status: true,
|
|
}
|
|
pageSize := int64(1000)
|
|
items := make([]idWithTime, 0)
|
|
for i := int64(0); i < 500; i++ {
|
|
skip := i * pageSize
|
|
list, _ := videocategorymod.FindManyVideoCategorys(ctx, filter, options.Find().SetSort(bson.M{videocategorymod.FN.VideoCategory.Id: -1}).SetSkip(skip).SetLimit(pageSize))
|
|
for _, info := range list {
|
|
if len(info.Tags) == 0 {
|
|
if !hasVideos {
|
|
continue
|
|
}
|
|
} else if !hasAnyID(info.Tags, usedTagIds) {
|
|
continue
|
|
}
|
|
if !videoCategoryHasContent(ctx, info) {
|
|
continue
|
|
}
|
|
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
|
}
|
|
if len(list) < int(pageSize) {
|
|
break
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
// videoCategoryContentFilter 与 app QueryVideoMore 使用相同的视频筛选条件。
|
|
func videoCategoryContentFilter(category *videocategorymod.VideoCategory) bson.M {
|
|
filter := bson.M{
|
|
videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK,
|
|
}
|
|
if category != nil && len(category.Tags) > 0 {
|
|
filter[videoinformationmod.FN.VideoInformation.Tags] = bson.M{"$in": category.Tags}
|
|
}
|
|
return filter
|
|
}
|
|
|
|
func videoCategoryHasContent(ctx context.Context, category *videocategorymod.VideoCategory) bool {
|
|
if category == nil || !category.Status {
|
|
return false
|
|
}
|
|
count, err := videoinformationmod.CountVideoInformations(ctx, videoCategoryContentFilter(category))
|
|
return err == nil && count > 0
|
|
}
|
|
|
|
func getActress(ctx context.Context) map[int64]idWithTime {
|
|
filter := bson.M{
|
|
actress.FN.Actress.Status: true,
|
|
}
|
|
pageSize := int64(1000)
|
|
pageIds := make(map[int64]idWithTime)
|
|
for i := int64(0); i < 100; i++ {
|
|
skip := i * pageSize
|
|
list, _ := actress.FindManyActresss(ctx, filter, options.Find().SetSkip(skip).SetLimit(pageSize).SetSort(bson.M{actress.FN.Actress.Id: -1}))
|
|
for _, info := range list {
|
|
pageIds[info.Id] = idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}
|
|
}
|
|
if len(list) < int(pageSize) {
|
|
break
|
|
}
|
|
}
|
|
return pageIds
|
|
}
|
|
|
|
func videoEntriesToIDWithTime(entries []videoEntry) []idWithTime {
|
|
items := make([]idWithTime, len(entries))
|
|
for i, e := range entries {
|
|
items[i] = idWithTime{Id: e.Id, Slug: e.Slug, LastMod: e.LastMod}
|
|
}
|
|
return items
|
|
}
|
|
|
|
func getVideoList(ctx context.Context) ([]videoEntry, map[int64]struct{}, map[int64]struct{}) {
|
|
filter := bson.M{videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK}
|
|
pageSize := int64(1000)
|
|
items := make([]videoEntry, 0)
|
|
tagIds := make(map[int64]struct{})
|
|
actressIds := make(map[int64]struct{})
|
|
for i := int64(0); i < 500; i++ {
|
|
skip := i * pageSize
|
|
list, _ := videoinformationmod.FindManyVideoInformations(ctx, filter, options.Find().SetSort(bson.M{videoinformationmod.FN.VideoInformation.Id: -1}).SetLimit(pageSize).SetSkip(skip))
|
|
for _, info := range list {
|
|
items = append(items, videoEntry{
|
|
Id: info.Id,
|
|
Slug: info.Slug,
|
|
Title: info.Title,
|
|
LastMod: info.UpdatedAt,
|
|
UnencryptedPreMp4: info.Video.UnencryptedPreMp4,
|
|
})
|
|
for _, tagId := range info.Tags {
|
|
tagIds[tagId] = struct{}{}
|
|
}
|
|
for _, actressId := range info.ActressIds {
|
|
if actressId > 0 {
|
|
actressIds[actressId] = struct{}{}
|
|
}
|
|
}
|
|
}
|
|
if len(list) < int(pageSize) {
|
|
break
|
|
}
|
|
}
|
|
return items, tagIds, actressIds
|
|
}
|
|
|
|
func getTranslation(config cfg.SysCfg, ctx context.Context) []string {
|
|
urlList := make([]string, 0)
|
|
wg := &sync.WaitGroup{}
|
|
wg.Add(2)
|
|
var translationCategoryList translationcategorymod.TranslationCategoryList
|
|
var translationList translationmod.TranslationInfoList
|
|
go func() {
|
|
defer wg.Done()
|
|
categoryFilter := bson.M{
|
|
translationcategorymod.FN.TranslationCategory.Status: true,
|
|
}
|
|
categoryOpt := &options.FindOptions{}
|
|
categoryOpt.SetLimit(1000).SetSort(bson.M{translationcategorymod.FN.TranslationCategory.Rank: -1})
|
|
|
|
translationCategoryList, _ = translationcategorymod.FindManyTranslationCategorys(ctx, categoryFilter, categoryOpt)
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
translationFilter := bson.M{
|
|
translationmod.FN.TranslationInfo.Status: true,
|
|
}
|
|
translationOpt := &options.FindOptions{}
|
|
translationOpt.SetLimit(1000).SetSort(bson.M{translationmod.FN.TranslationInfo.Rank: -1})
|
|
|
|
translationList, _ = translationmod.FindManyTranslationInfos(ctx, translationFilter, translationOpt)
|
|
}()
|
|
|
|
wg.Wait()
|
|
|
|
// 按 categoryId 建索引，避免嵌套遍历
|
|
translationByCategory := make(map[int64][]*translationmod.TranslationInfo, len(translationList))
|
|
for _, t := range translationList {
|
|
translationByCategory[t.TransCategoryId] = append(translationByCategory[t.TransCategoryId], t)
|
|
}
|
|
|
|
for _, categoryInfo := range translationCategoryList {
|
|
if categoryInfo.CategoryType == pubdef.TranslationTypeUrl {
|
|
if strings.HasPrefix(categoryInfo.UrlValue, "http://") || strings.HasPrefix(categoryInfo.UrlValue, "https://") {
|
|
continue
|
|
}
|
|
urlList = append(urlList, utils.JoinUrlPathAppendEnd(config.WebAddr, categoryInfo.UrlValue))
|
|
} else if categoryInfo.CategoryType == pubdef.TranslationTypeSub || categoryInfo.CategoryType == pubdef.TranslationTypeBottom {
|
|
for _, translationInfo := range translationByCategory[categoryInfo.Id] {
|
|
if translationInfo.IsFriendAds {
|
|
continue
|
|
} else if translationInfo.HtmlId > 0 {
|
|
title := strings.TrimSpace(translationInfo.Name)
|
|
if title != "" {
|
|
urlList = append(urlList, utils.JoinUrlPathKeepSlash(config.WebAddr, "/richtext?title="+url.QueryEscape(title)))
|
|
}
|
|
} else if len(translationInfo.Link) > 0 {
|
|
if !strings.HasPrefix(translationInfo.Link, "http://") && !strings.HasPrefix(translationInfo.Link, "https://") {
|
|
urlList = append(urlList, utils.JoinUrlPathAppendEnd(config.WebAddr, translationInfo.Link))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return urlList
|
|
}
|