Claude Opus 4.7
@@ -0,0 +1,49 @@
|
||||
package ipstat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
"videoweb/pub/common/enum/redisconst"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/common/utils"
|
||||
"videoweb/pub/models/ipdailystatmod"
|
||||
"videoweb/pub/pubdef"
|
||||
"videoweb/pub/redis"
|
||||
)
|
||||
|
||||
const syncInterval = 5 * time.Minute
|
||||
|
||||
func StartSync() {
|
||||
go func() {
|
||||
syncOnce()
|
||||
ticker := time.NewTicker(syncInterval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
syncOnce()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func syncOnce() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
now := time.Now().In(utils.LocBJ)
|
||||
appId := int64(pubdef.AppId)
|
||||
|
||||
for _, offset := range []int{0, -1} {
|
||||
date := now.AddDate(0, 0, offset).Format("20060102")
|
||||
key := redisconst.IpStatKey(appId, date)
|
||||
|
||||
count, err := redis.SCard(key)
|
||||
if err != nil {
|
||||
log.Warn("ipstat sync: SCard failed", log.Any("date", date), log.E(err))
|
||||
continue
|
||||
}
|
||||
|
||||
if err := ipdailystatmod.UpsertCount(ctx, appId, date, count); err != nil {
|
||||
log.Warn("ipstat sync: UpsertCount failed",
|
||||
log.Any("appId", appId), log.Any("date", date), log.E(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/pub/models/homeconfigmod"
|
||||
"videoweb/pub/models/playletchaptermod"
|
||||
"videoweb/pub/models/playletmod"
|
||||
"videoweb/pub/models/playlettagmod"
|
||||
"videoweb/pub/models/playletthememod"
|
||||
"videoweb/pub/seo/h5path"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
type playEpisode struct {
|
||||
Episode int64
|
||||
LastMod time.Time
|
||||
}
|
||||
|
||||
type idWithTime struct {
|
||||
Id int64
|
||||
Slug string
|
||||
LastMod time.Time
|
||||
IsOriginal bool
|
||||
Episodes []playEpisode
|
||||
}
|
||||
|
||||
type staticPageEntry struct {
|
||||
path string
|
||||
changeFreq string
|
||||
priority float32
|
||||
}
|
||||
|
||||
func buildStaticURLs(webAddr string, entries []staticPageEntry) []URL {
|
||||
urls := make([]URL, 0, len(entries))
|
||||
now := time.Now().Format(time.DateOnly)
|
||||
for _, entry := range entries {
|
||||
urls = append(urls, URL{
|
||||
Loc: h5path.SitemapAbs(webAddr, entry.path),
|
||||
LastMod: now,
|
||||
ChangeFreq: entry.changeFreq,
|
||||
Priority: entry.priority,
|
||||
})
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
func entityPath(prefix, slug string, id int64) string {
|
||||
return h5path.Entity(prefix, slug, id)
|
||||
}
|
||||
|
||||
func siteStaticPages() []staticPageEntry {
|
||||
return []staticPageEntry{
|
||||
{path: "/", changeFreq: "daily", priority: 1.0},
|
||||
{path: "/shorts", changeFreq: "daily", priority: 0.9},
|
||||
{path: "/shorts/popular", changeFreq: "daily", priority: 0.7},
|
||||
{path: "/shorts/original", changeFreq: "daily", priority: 0.7},
|
||||
{path: "/shorts/random", changeFreq: "daily", priority: 0.7},
|
||||
{path: "/series", changeFreq: "daily", priority: 0.9},
|
||||
{path: "/series/popular", changeFreq: "daily", priority: 0.7},
|
||||
{path: "/series/original", changeFreq: "daily", priority: 0.7},
|
||||
{path: "/series/random", changeFreq: "daily", priority: 0.7},
|
||||
{path: "/topics", changeFreq: "daily", priority: 0.8},
|
||||
{path: "/rank", changeFreq: "daily", priority: 0.8},
|
||||
{path: "/rank/new", changeFreq: "daily", priority: 0.7},
|
||||
{path: "/rank/binge", changeFreq: "daily", priority: 0.7},
|
||||
{path: "/home-road", changeFreq: "weekly", priority: 0.6},
|
||||
{path: "/apps", changeFreq: "weekly", priority: 0.6},
|
||||
{path: "/nav", changeFreq: "weekly", priority: 0.5},
|
||||
{path: "/info/about-us", changeFreq: "yearly", priority: 0.3},
|
||||
{path: "/info/user-agreement", changeFreq: "yearly", priority: 0.3},
|
||||
{path: "/info/privacy-policy", changeFreq: "yearly", priority: 0.3},
|
||||
{path: "/info/help-center", changeFreq: "yearly", priority: 0.3},
|
||||
{path: "/info/2257", changeFreq: "yearly", priority: 0.3},
|
||||
{path: "/info/dmca", changeFreq: "yearly", priority: 0.3},
|
||||
}
|
||||
}
|
||||
|
||||
// getURL 并发拉取短剧数据并写入 sitemap 分片。
|
||||
// 路径对齐 https://91dj-h5.80sygtm.com/ 上有独立 canonical、且未被 noindex 的页面。
|
||||
func getURL(sg *SitemapGenerator) {
|
||||
webAddr := strings.TrimSpace(sg.WebAddr)
|
||||
if webAddr == "" {
|
||||
return
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
listed listedPlaylets
|
||||
themeItems []idWithTime
|
||||
tagItems []idWithTime
|
||||
homeConfigs []homeConfigItem
|
||||
)
|
||||
|
||||
wg.Add(4)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
listed = getPlayletList(ctx)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
themeItems = getThemeList(ctx)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
tagItems = getTagList(ctx)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
homeConfigs = getHomeConfigList(ctx)
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
// 题材/标签没有上架短剧时不写入，避免空列表页被收录。
|
||||
themeItems = keepWithListedPlaylets(themeItems, listed.themeIDs)
|
||||
tagItems = keepWithListedPlaylets(tagItems, listed.tagIDs)
|
||||
|
||||
pageURLs := buildStaticURLs(webAddr, siteStaticPages())
|
||||
for _, item := range themeItems {
|
||||
pageURLs = append(pageURLs, newURL(webAddr, h5path.TopicDetail(item.Slug, item.Id), item.LastMod, "daily", 0.8))
|
||||
}
|
||||
for _, item := range tagItems {
|
||||
if path := h5path.Tag(item.Slug, item.Id); path != "" {
|
||||
pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "weekly", 0.6))
|
||||
}
|
||||
}
|
||||
for _, item := range homeConfigs {
|
||||
path := ""
|
||||
switch item.Type {
|
||||
case homeconfigmod.TypePlaylet:
|
||||
path = h5path.MoreList(item.Slug, item.Id)
|
||||
}
|
||||
if path != "" {
|
||||
pageURLs = append(pageURLs, newURL(webAddr, path, item.LastMod, "daily", 0.7))
|
||||
}
|
||||
}
|
||||
if len(pageURLs) > 0 {
|
||||
_ = sg.writeSitemapFile("sitemap-pages.xml", pageURLs)
|
||||
}
|
||||
|
||||
// 详情页 /short-drama/{slug} 与播放页 /short-drama/{slug}/play/{集数} 都有独立 canonical。
|
||||
// 原创且该集没有水印成片时不提交播放页，避免空壳播放地址被收录。
|
||||
populatePlayEpisodes(ctx, listed.items)
|
||||
if urls := buildPlayletPathURLs(webAddr, listed.items, "weekly", 0.8); len(urls) > 0 {
|
||||
_ = sg.writeSitemapFiles("sitemap-playletdetail", urls)
|
||||
}
|
||||
if urls := buildPlayletEpisodeURLs(webAddr, listed.items, "weekly", 0.6); len(urls) > 0 {
|
||||
_ = sg.writeSitemapFiles("sitemap-playletplay", urls)
|
||||
}
|
||||
_ = sg.writeSitemapFiles("sitemap-seriesplay", nil)
|
||||
}
|
||||
|
||||
type listedPlaylets struct {
|
||||
items []idWithTime
|
||||
themeIDs map[int64]struct{}
|
||||
tagIDs map[int64]struct{}
|
||||
}
|
||||
|
||||
type homeConfigItem struct {
|
||||
Id int64
|
||||
Slug string
|
||||
Type int32
|
||||
LastMod time.Time
|
||||
}
|
||||
|
||||
func newURL(webAddr, path string, lastMod time.Time, changeFreq string, priority float32) URL {
|
||||
mod := formatLastMod(lastMod)
|
||||
if mod == "" {
|
||||
mod = time.Now().Format(time.DateOnly)
|
||||
}
|
||||
return URL{
|
||||
Loc: h5path.SitemapAbs(webAddr, path),
|
||||
LastMod: mod,
|
||||
ChangeFreq: changeFreq,
|
||||
Priority: priority,
|
||||
}
|
||||
}
|
||||
|
||||
func keepWithListedPlaylets(items []idWithTime, listed map[int64]struct{}) []idWithTime {
|
||||
if len(items) == 0 || len(listed) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]idWithTime, 0, len(items))
|
||||
for _, item := range items {
|
||||
if _, ok := listed[item.Id]; ok {
|
||||
out = append(out, item)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func addListedIDs(dst map[int64]struct{}, ids []int64) {
|
||||
for _, id := range ids {
|
||||
if id > 0 {
|
||||
dst[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func buildPlayletPathURLs(webAddr string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
urls := make([]URL, 0, len(items))
|
||||
for _, item := range items {
|
||||
path := h5path.Playlet(item.Slug, item.Id)
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
urls = append(urls, newURL(webAddr, path, item.LastMod, changeFreq, priority))
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
func buildPlayletEpisodeURLs(webAddr string, items []idWithTime, changeFreq string, priority float32) []URL {
|
||||
urls := make([]URL, 0)
|
||||
for _, item := range items {
|
||||
for _, episode := range item.Episodes {
|
||||
path := h5path.PlayletPlay(item.Slug, item.Id, episode.Episode)
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
lastMod := episode.LastMod
|
||||
if lastMod.IsZero() {
|
||||
lastMod = item.LastMod
|
||||
}
|
||||
urls = append(urls, newURL(webAddr, path, lastMod, changeFreq, priority))
|
||||
}
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
func skipOriginalChapterWithoutWatermark(isOriginal bool, watermarkURL string) bool {
|
||||
return isOriginal && strings.TrimSpace(watermarkURL) == ""
|
||||
}
|
||||
|
||||
// populatePlayEpisodes 收集上架剧集的播放页。原创且没有水印成片的集不提交。
|
||||
func populatePlayEpisodes(ctx context.Context, items []idWithTime) {
|
||||
for start := 0; start < len(items); start += 500 {
|
||||
end := start + 500
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
ids := make([]int64, 0, end-start)
|
||||
byID := make(map[int64]*idWithTime, end-start)
|
||||
for i := start; i < end; i++ {
|
||||
if items[i].Id <= 0 {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, items[i].Id)
|
||||
byID[items[i].Id] = &items[i]
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
continue
|
||||
}
|
||||
filter := bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.PlayletId: bson.M{"$in": ids},
|
||||
}
|
||||
for key, value := range playletchaptermod.VisibleStatusFilter() {
|
||||
filter[key] = value
|
||||
}
|
||||
chapters, err := playletchaptermod.FindManyPlayletChapter(ctx, filter, options.Find().
|
||||
SetSort(bson.D{
|
||||
{Key: playletchaptermod.FN.PlayletChapter.CurrentEpisode, Value: 1},
|
||||
{Key: playletchaptermod.FN.PlayletChapter.Id, Value: 1},
|
||||
}).
|
||||
SetProjection(bson.M{
|
||||
playletchaptermod.FN.PlayletChapter.Id: 1,
|
||||
playletchaptermod.FN.PlayletChapter.PlayletId: 1,
|
||||
playletchaptermod.FN.PlayletChapter.CurrentEpisode: 1,
|
||||
playletchaptermod.FN.PlayletChapter.WatermarkVideoUrl: 1,
|
||||
playletchaptermod.FN.PlayletChapter.UpdatedAt: 1,
|
||||
}))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
seen := make(map[int64]map[int64]struct{}, len(ids))
|
||||
for _, ch := range chapters {
|
||||
if ch == nil || ch.PlayletId <= 0 || ch.CurrentEpisode <= 0 {
|
||||
continue
|
||||
}
|
||||
item := byID[ch.PlayletId]
|
||||
if item == nil || skipOriginalChapterWithoutWatermark(item.IsOriginal, ch.WatermarkVideoUrl) {
|
||||
continue
|
||||
}
|
||||
if seen[ch.PlayletId] == nil {
|
||||
seen[ch.PlayletId] = make(map[int64]struct{})
|
||||
}
|
||||
if _, ok := seen[ch.PlayletId][ch.CurrentEpisode]; ok {
|
||||
continue
|
||||
}
|
||||
seen[ch.PlayletId][ch.CurrentEpisode] = struct{}{}
|
||||
item.Episodes = append(item.Episodes, playEpisode{Episode: ch.CurrentEpisode, LastMod: ch.UpdatedAt})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func formatLastMod(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return t.Format(time.DateOnly)
|
||||
}
|
||||
|
||||
func getPlayletList(ctx context.Context) listedPlaylets {
|
||||
result := listedPlaylets{
|
||||
items: make([]idWithTime, 0),
|
||||
themeIDs: make(map[int64]struct{}),
|
||||
tagIDs: make(map[int64]struct{}),
|
||||
}
|
||||
filter := bson.M{
|
||||
playletmod.FN.Playlet.Status: playletmod.StatusUp,
|
||||
}
|
||||
pageSize := int64(1000)
|
||||
for i := int64(0); i < 500; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := playletmod.FindManyPlaylet(ctx, filter, options.Find().
|
||||
SetSort(bson.M{playletmod.FN.Playlet.Id: -1}).
|
||||
SetSkip(skip).
|
||||
SetLimit(pageSize).
|
||||
SetProjection(bson.M{
|
||||
playletmod.FN.Playlet.Id: 1,
|
||||
playletmod.FN.Playlet.Slug: 1,
|
||||
playletmod.FN.Playlet.IsOriginal: 1,
|
||||
playletmod.FN.Playlet.UpdatedAt: 1,
|
||||
playletmod.FN.Playlet.OnlineAt: 1,
|
||||
playletmod.FN.Playlet.ThemeIds: 1,
|
||||
playletmod.FN.Playlet.TagIds: 1,
|
||||
}))
|
||||
for _, info := range list {
|
||||
if info == nil || info.Id <= 0 {
|
||||
continue
|
||||
}
|
||||
lastMod := info.UpdatedAt
|
||||
if lastMod.IsZero() {
|
||||
lastMod = info.OnlineAt
|
||||
}
|
||||
result.items = append(result.items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: lastMod, IsOriginal: info.IsOriginal})
|
||||
addListedIDs(result.themeIDs, info.ThemeIds)
|
||||
addListedIDs(result.tagIDs, info.TagIds)
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func getThemeList(ctx context.Context) []idWithTime {
|
||||
filter := bson.M{
|
||||
playletthememod.FN.PlayletTheme.Status: true,
|
||||
}
|
||||
pageSize := int64(1000)
|
||||
items := make([]idWithTime, 0)
|
||||
for i := int64(0); i < 100; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := playletthememod.FindManyPlayletTheme(ctx, filter, options.Find().
|
||||
SetSort(bson.M{playletthememod.FN.PlayletTheme.Rank: -1}).
|
||||
SetSkip(skip).
|
||||
SetLimit(pageSize))
|
||||
for _, info := range list {
|
||||
if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" {
|
||||
continue
|
||||
}
|
||||
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func getTagList(ctx context.Context) []idWithTime {
|
||||
filter := bson.M{
|
||||
playlettagmod.FN.PlayletTag.Status: true,
|
||||
}
|
||||
pageSize := int64(1000)
|
||||
items := make([]idWithTime, 0)
|
||||
for i := int64(0); i < 100; i++ {
|
||||
skip := i * pageSize
|
||||
list, _ := playlettagmod.FindManyPlayletTag(ctx, filter, options.Find().
|
||||
SetSort(bson.M{playlettagmod.FN.PlayletTag.Rank: -1}).
|
||||
SetSkip(skip).
|
||||
SetLimit(pageSize))
|
||||
for _, info := range list {
|
||||
if info == nil || info.Id <= 0 || strings.TrimSpace(info.Name) == "" {
|
||||
continue
|
||||
}
|
||||
items = append(items, idWithTime{Id: info.Id, Slug: info.Slug, LastMod: info.UpdatedAt})
|
||||
}
|
||||
if len(list) < int(pageSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func getHomeConfigList(ctx context.Context) []homeConfigItem {
|
||||
list, _ := homeconfigmod.FindManyHomeConfig(ctx, bson.M{
|
||||
homeconfigmod.FN.HomeConfig.Status: true,
|
||||
homeconfigmod.FN.HomeConfig.Type: bson.M{
|
||||
"$in": []int32{homeconfigmod.TypePlaylet},
|
||||
},
|
||||
}, options.Find().SetSort(bson.M{homeconfigmod.FN.HomeConfig.Rank: -1}))
|
||||
items := make([]homeConfigItem, 0, len(list))
|
||||
for _, info := range list {
|
||||
if info == nil || info.Id <= 0 {
|
||||
continue
|
||||
}
|
||||
items = append(items, homeConfigItem{Id: info.Id, Slug: info.Slug, Type: info.Type, LastMod: info.UpdatedAt})
|
||||
}
|
||||
return items
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"videoweb/app/service/configser"
|
||||
"videoweb/pub/common/log"
|
||||
)
|
||||
|
||||
var generateMu sync.Mutex
|
||||
|
||||
// GenerateSitemap 生成全站 sitemap 分片及索引文件。
|
||||
func GenerateSitemap() {
|
||||
generateMu.Lock()
|
||||
defer generateMu.Unlock()
|
||||
|
||||
webAddr := strings.TrimSpace(configser.GetConfigInfo().WebAddr)
|
||||
if webAddr == "" {
|
||||
log.Warn("GenerateSitemap: WebAddr is empty, skip")
|
||||
return
|
||||
}
|
||||
|
||||
generator := NewSitemapGenerator(webAddr, OutputDir)
|
||||
if err := generator.Generate(); err != nil {
|
||||
log.Error("GenerateSitemap failed", log.E(err))
|
||||
return
|
||||
}
|
||||
if err := writeRobotsTXT(webAddr); err != nil {
|
||||
log.Error("GenerateSitemap write robots.txt failed", log.E(err))
|
||||
return
|
||||
}
|
||||
log.Info("GenerateSitemap done",
|
||||
log.Any("files", len(generator.SitemapFiles)),
|
||||
log.Any("webAddr", webAddr))
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
"videoweb/pub/common/log"
|
||||
)
|
||||
|
||||
func StartSitemapJob(interval time.Duration, outDir string) {
|
||||
if interval <= 0 {
|
||||
interval = 6 * time.Hour
|
||||
}
|
||||
if outDir == "" {
|
||||
outDir = OutputDir
|
||||
}
|
||||
|
||||
go func() {
|
||||
if mkErr := os.MkdirAll(outDir, 0755); mkErr != nil {
|
||||
log.Error("StartSitemapJob mkdir failed", log.E(mkErr))
|
||||
return
|
||||
}
|
||||
if mkErr := os.MkdirAll(PageOutputDir, 0755); mkErr != nil {
|
||||
log.Error("StartSitemapJob mkdir page dir failed", log.E(mkErr))
|
||||
return
|
||||
}
|
||||
|
||||
GenerateSitemap()
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
GenerateSitemap()
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
stylesheetIndex = `<?xml-stylesheet type="text/xsl" href="/static/sitemapindex.xsl"?>` + "\n"
|
||||
stylesheetURLSet = `<?xml-stylesheet type="text/xsl" href="/static/sitemap.xsl"?>` + "\n"
|
||||
|
||||
defaultMaxURLsPerFile = 40000
|
||||
|
||||
// OutputDir 分片 sitemap 输出目录（相对 app 进程 cwd）。
|
||||
// 对齐 jha-1045-hgdjnet-web：/server/web_91ys/h5/template/pages/sitemaps
|
||||
OutputDir = "../h5/template/pages/sitemaps"
|
||||
// PageOutputDir sitemap 索引输出目录，供 H5 /sitemap.xml 读取
|
||||
PageOutputDir = "../h5/template/pages"
|
||||
)
|
||||
|
||||
type URL struct {
|
||||
Loc string `xml:"loc"`
|
||||
LastMod string `xml:"lastmod"`
|
||||
ChangeFreq string `xml:"changefreq"`
|
||||
Priority float32 `xml:"priority"`
|
||||
}
|
||||
|
||||
type Sitemap struct {
|
||||
XMLName xml.Name `xml:"urlset"`
|
||||
Xmlns string `xml:"xmlns,attr"`
|
||||
URLs []URL `xml:"url"`
|
||||
}
|
||||
|
||||
type SitemapIndex struct {
|
||||
XMLName xml.Name `xml:"sitemapindex"`
|
||||
Xmlns string `xml:"xmlns,attr"`
|
||||
Sitemaps []SitemapIndexEntry `xml:"sitemap"`
|
||||
}
|
||||
|
||||
type SitemapIndexEntry struct {
|
||||
Loc string `xml:"loc"`
|
||||
LastMod string `xml:"lastmod"`
|
||||
}
|
||||
|
||||
type SitemapGenerator struct {
|
||||
MaxURLsPerFile int
|
||||
OutputDir string
|
||||
WebAddr string
|
||||
SitemapFiles []string
|
||||
}
|
||||
|
||||
func NewSitemapGenerator(webAddr, outputDir string) *SitemapGenerator {
|
||||
return &SitemapGenerator{
|
||||
MaxURLsPerFile: defaultMaxURLsPerFile,
|
||||
OutputDir: outputDir,
|
||||
WebAddr: webAddr,
|
||||
SitemapFiles: make([]string, 0),
|
||||
}
|
||||
}
|
||||
|
||||
func (sg *SitemapGenerator) maxURLsPerFile() int {
|
||||
if sg.MaxURLsPerFile > 0 {
|
||||
return sg.MaxURLsPerFile
|
||||
}
|
||||
return defaultMaxURLsPerFile
|
||||
}
|
||||
|
||||
func (sg *SitemapGenerator) Generate() error {
|
||||
sg.SitemapFiles = make([]string, 0)
|
||||
|
||||
if err := os.MkdirAll(sg.OutputDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(PageOutputDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
getURL(sg)
|
||||
|
||||
if err := sg.generateSitemapIndex(); err != nil {
|
||||
return err
|
||||
}
|
||||
return copySitemapIndexToPageRoot()
|
||||
}
|
||||
|
||||
func (sg *SitemapGenerator) writeSitemapFile(filename string, urls []URL) error {
|
||||
sitemap := Sitemap{
|
||||
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||||
URLs: urls,
|
||||
}
|
||||
|
||||
filePath := filepath.Join(sg.OutputDir, filename)
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if _, err := file.WriteString(xml.Header); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := file.WriteString(stylesheetURLSet); err != nil {
|
||||
return err
|
||||
}
|
||||
encoder := xml.NewEncoder(file)
|
||||
encoder.Indent("", " ")
|
||||
|
||||
if err := encoder.Encode(sitemap); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sg.SitemapFiles = append(sg.SitemapFiles, filename)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sg *SitemapGenerator) writeSitemapFiles(baseName string, urls []URL) error {
|
||||
baseName = strings.TrimSuffix(strings.TrimSpace(baseName), ".xml")
|
||||
if baseName == "" {
|
||||
return nil
|
||||
}
|
||||
if len(urls) == 0 {
|
||||
sg.cleanupSitemapShards(baseName, 0)
|
||||
return nil
|
||||
}
|
||||
|
||||
limit := sg.maxURLsPerFile()
|
||||
shardCount := (len(urls) + limit - 1) / limit
|
||||
if shardCount == 1 {
|
||||
sg.cleanupSitemapShards(baseName, 1)
|
||||
return sg.writeSitemapFile(baseName+".xml", urls)
|
||||
}
|
||||
|
||||
sg.cleanupSitemapShards(baseName, shardCount)
|
||||
for i := 0; i < shardCount; i++ {
|
||||
start := i * limit
|
||||
end := start + limit
|
||||
if end > len(urls) {
|
||||
end = len(urls)
|
||||
}
|
||||
filename := fmt.Sprintf("%s-%d.xml", baseName, i)
|
||||
if err := sg.writeSitemapFile(filename, urls[start:end]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sg *SitemapGenerator) cleanupSitemapShards(baseName string, shardCount int) {
|
||||
entries, err := os.ReadDir(sg.OutputDir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
prefix := baseName + "-"
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ".xml") {
|
||||
idxText := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".xml")
|
||||
idx, err := strconv.Atoi(idxText)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if shardCount <= 1 || idx >= shardCount {
|
||||
_ = os.Remove(filepath.Join(sg.OutputDir, name))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if shardCount > 1 && name == baseName+".xml" {
|
||||
_ = os.Remove(filepath.Join(sg.OutputDir, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sg *SitemapGenerator) generateSitemapIndex() error {
|
||||
index := SitemapIndex{
|
||||
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||||
Sitemaps: make([]SitemapIndexEntry, 0, len(sg.SitemapFiles)),
|
||||
}
|
||||
|
||||
now := time.Now().Format(time.DateOnly)
|
||||
for _, filename := range sg.SitemapFiles {
|
||||
index.Sitemaps = append(index.Sitemaps, SitemapIndexEntry{
|
||||
Loc: fmt.Sprintf("%s/sitemaps/%s", strings.TrimSuffix(sg.WebAddr, "/"), filename),
|
||||
LastMod: now,
|
||||
})
|
||||
}
|
||||
|
||||
filePath := filepath.Join(sg.OutputDir, "sitemap.xml")
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if _, err := file.WriteString(xml.Header); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := file.WriteString(stylesheetIndex); err != nil {
|
||||
return err
|
||||
}
|
||||
encoder := xml.NewEncoder(file)
|
||||
encoder.Indent("", " ")
|
||||
|
||||
return encoder.Encode(index)
|
||||
}
|
||||
|
||||
func copySitemapIndexToPageRoot() error {
|
||||
src := filepath.Join(OutputDir, "sitemap.xml")
|
||||
dst := filepath.Join(PageOutputDir, "sitemap.xml")
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(dst, data, 0644)
|
||||
}
|
||||
|
||||
func writeRobotsTXT(webAddr string) error {
|
||||
body := buildRobotsTXT(webAddr)
|
||||
if body == "" {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(PageOutputDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(PageOutputDir, "robots.txt"), []byte(body), 0644)
|
||||
}
|
||||
|
||||
func buildRobotsTXT(webAddr string) string {
|
||||
base := strings.TrimSuffix(strings.TrimSpace(webAddr), "/")
|
||||
if base == "" {
|
||||
return ""
|
||||
}
|
||||
body := "User-agent: *\nAllow: /\n\n" +
|
||||
"# 非 HTML 接口与播放器工具无需被搜索引擎抓取。\n" +
|
||||
"Disallow: /api/\n" +
|
||||
"Disallow: /h5/\n" +
|
||||
"Disallow: /ui/\n" +
|
||||
"Disallow: /healthz\n" +
|
||||
"Disallow: /enkey\n" +
|
||||
"Disallow: /newenkey\n" +
|
||||
"Disallow: /cdn-cgi/\n\n" +
|
||||
"# 搜索、登录和账户 HTML 保持可抓取，以便爬虫读取页面内 noindex。\n\n" +
|
||||
"Sitemap: " + base + "/sitemap.xml\n"
|
||||
if host := hostFromWebAddr(base); host != "" {
|
||||
body += "Host: " + host + "\n"
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func hostFromWebAddr(webAddr string) string {
|
||||
webAddr = strings.TrimSpace(webAddr)
|
||||
webAddr = strings.TrimPrefix(webAddr, "https://")
|
||||
webAddr = strings.TrimPrefix(webAddr, "http://")
|
||||
if i := strings.IndexByte(webAddr, '/'); i >= 0 {
|
||||
webAddr = webAddr[:i]
|
||||
}
|
||||
return webAddr
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package seo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
"videoweb/app/skd/job/sitemap"
|
||||
"videoweb/pub/common/log"
|
||||
"videoweb/pub/seo/sitemapdirty"
|
||||
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
var seoCron *cron.Cron
|
||||
|
||||
func init() {
|
||||
seoCron = cron.New(cron.WithSeconds())
|
||||
}
|
||||
|
||||
var (
|
||||
dirtyMu sync.Mutex
|
||||
// generatedDirtyAt 最近一次 sitemap 生成所对应的内容变更时间
|
||||
generatedDirtyAt time.Time
|
||||
)
|
||||
|
||||
// Start 启动 SEO 静态资源生成（sitemap）。
|
||||
// 页面级搜索引擎推送由 web 进程的 seopush 任务负责，不在这里处理。
|
||||
func Start() {
|
||||
generatedDirtyAt = sitemapdirty.LastDirtyAt(context.Background())
|
||||
sitemap.StartSitemapJob(6*time.Hour, sitemap.OutputDir)
|
||||
|
||||
// 短剧发布/更新/下架发生在 web 进程，这里检查它留下的变更标记，有变更才重新生成
|
||||
_, _ = seoCron.AddFunc("0 */2 * * * ?", func() {
|
||||
regenerateSitemapIfDirty()
|
||||
})
|
||||
seoCron.Start()
|
||||
log.Info("seo skd started")
|
||||
}
|
||||
|
||||
func regenerateSitemapIfDirty() {
|
||||
if !dirtyMu.TryLock() {
|
||||
return
|
||||
}
|
||||
defer dirtyMu.Unlock()
|
||||
|
||||
dirtyAt := sitemapdirty.LastDirtyAt(context.Background())
|
||||
if dirtyAt.IsZero() || !dirtyAt.After(generatedDirtyAt) {
|
||||
return
|
||||
}
|
||||
log.Info("seo: content changed, regenerate sitemap", log.Any("dirtyAt", dirtyAt))
|
||||
sitemap.GenerateSitemap()
|
||||
generatedDirtyAt = dirtyAt
|
||||
}
|
||||
|
||||
func Stop() {
|
||||
if seoCron != nil {
|
||||
seoCron.Stop()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user