Initial commit: web_hghl_server
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,312 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/pub/common/utils"
|
||||
"videoweb/pub/models/actress"
|
||||
"videoweb/pub/models/cfg"
|
||||
"videoweb/pub/models/translationcategorymod"
|
||||
"videoweb/pub/models/translationmod"
|
||||
"videoweb/pub/models/videocategorymod"
|
||||
"videoweb/pub/models/videoinformationmod"
|
||||
"videoweb/pub/pubdef"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// idWithTime 携带 ID、slug 和实际更新时间
|
||||
type idWithTime struct {
|
||||
Id int64
|
||||
Slug string
|
||||
LastMod time.Time
|
||||
}
|
||||
|
||||
func hasAnyID(ids []int64, used map[int64]struct{}) bool {
|
||||
for _, id := range ids {
|
||||
if _, ok := used[id]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isSitemapChannelPath(path, prefix string, extras ...string) bool {
|
||||
if path == prefix || strings.HasPrefix(path, prefix+"/") {
|
||||
return true
|
||||
}
|
||||
for _, extra := range extras {
|
||||
if path == extra {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func detailSegment(slug string, id int64) string {
|
||||
if s := strings.TrimSpace(slug); s != "" {
|
||||
return s
|
||||
}
|
||||
if id > 0 {
|
||||
return strconv.FormatInt(id, 10)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func buildChannelKeyURLs(webAddr, channel, keyPrefix string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
urls := make([]URL, 0, len(items))
|
||||
channel = strings.TrimRight(channel, "/")
|
||||
for _, item := range items {
|
||||
seg := detailSegment(item.Slug, item.Id)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
lastMod := item.LastMod.Format(time.DateOnly)
|
||||
if item.LastMod.IsZero() {
|
||||
lastMod = time.Now().Format(time.DateOnly)
|
||||
}
|
||||
urls = append(urls, URL{
|
||||
Loc: utils.JoinUrlPathKeepSlash(webAddr, channel+"/"+keyPrefix+seg),
|
||||
LastMod: lastMod,
|
||||
ChangeFreq: changeFreq,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
// buildCategoryURLs 分类页:/{channel}/c-{slug},无 slug 时回退 c-{id}。
|
||||
func buildCategoryURLs(webAddr, channel string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
return buildChannelKeyURLs(webAddr, channel, "c-", items, changeFreq, priority)
|
||||
}
|
||||
|
||||
// buildTagURLs 标签页:/{channel}/t-{slug},无 slug 时回退 t-{id}。
|
||||
func buildTagURLs(webAddr, channel string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
return buildChannelKeyURLs(webAddr, channel, "t-", items, changeFreq, priority)
|
||||
}
|
||||
|
||||
// buildDetailURLs 详情页:/{prefix}/{slug},无 slug 时回退数字 id。
|
||||
func buildDetailURLs(webAddr, prefix string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
urls := make([]URL, 0, len(items))
|
||||
for _, item := range items {
|
||||
seg := detailSegment(item.Slug, item.Id)
|
||||
if seg == "" {
|
||||
continue
|
||||
}
|
||||
lastMod := item.LastMod.Format(time.DateOnly)
|
||||
if item.LastMod.IsZero() {
|
||||
lastMod = time.Now().Format(time.DateOnly)
|
||||
}
|
||||
urls = append(urls, URL{
|
||||
Loc: utils.JoinUrlPathKeepSlash(webAddr, prefix+"/"+seg),
|
||||
LastMod: lastMod,
|
||||
ChangeFreq: changeFreq,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
// buildURLs 根据 ID+时间列表批量构建 URL 条目
|
||||
func buildURLs(webAddr string, pathFmt string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
urls := make([]URL, 0, len(items))
|
||||
for _, item := range items {
|
||||
lastMod := item.LastMod.Format(time.DateOnly)
|
||||
if item.LastMod.IsZero() {
|
||||
lastMod = time.Now().Format(time.DateOnly)
|
||||
}
|
||||
urls = append(urls, URL{
|
||||
Loc: utils.JoinUrlPathKeepSlash(webAddr, fmt.Sprintf(pathFmt, item.Id)),
|
||||
LastMod: lastMod,
|
||||
ChangeFreq: changeFreq,
|
||||
Priority: priority,
|
||||
})
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
// filterIdsWithTime 返回 source 中存在于 validMap 的条目(含 slug)。
|
||||
func filterIdsWithTime(source map[int64]struct{}, validMap map[int64]idWithTime) []idWithTime {
|
||||
result := make([]idWithTime, 0, len(source))
|
||||
for id := range source {
|
||||
if item, ok := validMap[id]; ok {
|
||||
result = append(result, item)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func getVideoCategoryList(ctx context.Context, usedTagIds map[int64]struct{}, hasVideos bool) []idWithTime {
|
||||
filter := bson.M{
|
||||
videocategorymod.FN.VideoCategory.Status: true,
|
||||
}
|
||||
pageSize := int64(1000)
|
||||
items := make([]idWithTime, 0)
|
||||
for i := int64(0); i < 500; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := videocategorymod.FindManyVideoCategorys(ctx, filter, options.Find().SetSort(bson.M{videocategorymod.FN.VideoCategory.Id: -1}).SetSkip(skip).SetLimit(pageSize))
|
||||
for _, info := range list {
|
||||
if len(info.Tags) == 0 {
|
||||
if !hasVideos {
|
||||
continue
|
||||
}
|
||||
} else if !hasAnyID(info.Tags, usedTagIds) {
|
||||
continue
|
||||
}
|
||||
if !videoCategoryHasContent(ctx, info) {
|
||||
continue
|
||||
}
|
||||
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// videoCategoryContentFilter 与 app QueryVideoMore 使用相同的视频筛选条件。
|
||||
func videoCategoryContentFilter(category *videocategorymod.VideoCategory) bson.M {
|
||||
filter := bson.M{
|
||||
videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK,
|
||||
}
|
||||
if category != nil && len(category.Tags) > 0 {
|
||||
filter[videoinformationmod.FN.VideoInformation.Tags] = bson.M{"$in": category.Tags}
|
||||
}
|
||||
return filter
|
||||
}
|
||||
|
||||
func videoCategoryHasContent(ctx context.Context, category *videocategorymod.VideoCategory) bool {
|
||||
if category == nil || !category.Status {
|
||||
return false
|
||||
}
|
||||
count, err := videoinformationmod.CountVideoInformations(ctx, videoCategoryContentFilter(category))
|
||||
return err == nil && count > 0
|
||||
}
|
||||
|
||||
func getActress(ctx context.Context) map[int64]idWithTime {
|
||||
filter := bson.M{
|
||||
actress.FN.Actress.Status: true,
|
||||
}
|
||||
pageSize := int64(1000)
|
||||
pageIds := make(map[int64]idWithTime)
|
||||
for i := int64(0); i < 100; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := actress.FindManyActresss(ctx, filter, options.Find().SetSkip(skip).SetLimit(pageSize).SetSort(bson.M{actress.FN.Actress.Id: -1}))
|
||||
for _, info := range list {
|
||||
pageIds[info.Id] = idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt}
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return pageIds
|
||||
}
|
||||
|
||||
func videoEntriesToIDWithTime(entries []videoEntry) []idWithTime {
|
||||
items := make([]idWithTime, len(entries))
|
||||
for i, e := range entries {
|
||||
items[i] = idWithTime{Id: e.Id, Slug: e.Slug, LastMod: e.LastMod}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func getVideoList(ctx context.Context) ([]videoEntry, map[int64]struct{}, map[int64]struct{}) {
|
||||
filter := bson.M{videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK}
|
||||
pageSize := int64(1000)
|
||||
items := make([]videoEntry, 0)
|
||||
tagIds := make(map[int64]struct{})
|
||||
actressIds := make(map[int64]struct{})
|
||||
for i := int64(0); i < 500; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := videoinformationmod.FindManyVideoInformations(ctx, filter, options.Find().SetSort(bson.M{videoinformationmod.FN.VideoInformation.Id: -1}).SetLimit(pageSize).SetSkip(skip))
|
||||
for _, info := range list {
|
||||
items = append(items, videoEntry{
|
||||
Id: info.Id,
|
||||
Slug: info.Slug,
|
||||
Title: info.Title,
|
||||
LastMod: info.UpdatedAt,
|
||||
UnencryptedPreMp4: info.Video.UnencryptedPreMp4,
|
||||
})
|
||||
for _, tagId := range info.Tags {
|
||||
tagIds[tagId] = struct{}{}
|
||||
}
|
||||
for _, actressId := range info.ActressIds {
|
||||
if actressId > 0 {
|
||||
actressIds[actressId] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return items, tagIds, actressIds
|
||||
}
|
||||
|
||||
func getTranslation(config cfg.SysCfg, ctx context.Context) []string {
|
||||
urlList := make([]string, 0)
|
||||
wg := &sync.WaitGroup{}
|
||||
wg.Add(2)
|
||||
var translationCategoryList translationcategorymod.TranslationCategoryList
|
||||
var translationList translationmod.TranslationInfoList
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
categoryFilter := bson.M{
|
||||
translationcategorymod.FN.TranslationCategory.Status: true,
|
||||
}
|
||||
categoryOpt := &options.FindOptions{}
|
||||
categoryOpt.SetLimit(1000).SetSort(bson.M{translationcategorymod.FN.TranslationCategory.Rank: -1})
|
||||
|
||||
translationCategoryList, _ = translationcategorymod.FindManyTranslationCategorys(ctx, categoryFilter, categoryOpt)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
translationFilter := bson.M{
|
||||
translationmod.FN.TranslationInfo.Status: true,
|
||||
}
|
||||
translationOpt := &options.FindOptions{}
|
||||
translationOpt.SetLimit(1000).SetSort(bson.M{translationmod.FN.TranslationInfo.Rank: -1})
|
||||
|
||||
translationList, _ = translationmod.FindManyTranslationInfos(ctx, translationFilter, translationOpt)
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// 按 categoryId 建索引,避免嵌套遍历
|
||||
translationByCategory := make(map[int64][]*translationmod.TranslationInfo, len(translationList))
|
||||
for _, t := range translationList {
|
||||
translationByCategory[t.TransCategoryId] = append(translationByCategory[t.TransCategoryId], t)
|
||||
}
|
||||
|
||||
for _, categoryInfo := range translationCategoryList {
|
||||
if categoryInfo.CategoryType == pubdef.TranslationTypeUrl {
|
||||
if strings.HasPrefix(categoryInfo.UrlValue, "http://") || strings.HasPrefix(categoryInfo.UrlValue, "https://") {
|
||||
continue
|
||||
}
|
||||
urlList = append(urlList, utils.JoinUrlPathAppendEnd(config.WebAddr, categoryInfo.UrlValue))
|
||||
} else if categoryInfo.CategoryType == pubdef.TranslationTypeSub || categoryInfo.CategoryType == pubdef.TranslationTypeBottom {
|
||||
for _, translationInfo := range translationByCategory[categoryInfo.Id] {
|
||||
if translationInfo.IsFriendAds {
|
||||
continue
|
||||
} else if translationInfo.HtmlId > 0 {
|
||||
title := strings.TrimSpace(translationInfo.Name)
|
||||
if title != "" {
|
||||
urlList = append(urlList, utils.JoinUrlPathKeepSlash(config.WebAddr, "/richtext?title="+url.QueryEscape(title)))
|
||||
}
|
||||
} else if len(translationInfo.Link) > 0 {
|
||||
if !strings.HasPrefix(translationInfo.Link, "http://") && !strings.HasPrefix(translationInfo.Link, "https://") {
|
||||
urlList = append(urlList, utils.JoinUrlPathAppendEnd(config.WebAddr, translationInfo.Link))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return urlList
|
||||
}
|
||||
Reference in New Issue
Block a user