246 lines
6.6 KiB
Go
246 lines
6.6 KiB
Go
package sitemap
|
||
|
||
import (
|
||
"encoding/xml"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
const (
|
||
// 浏览器用 XSLT 把 loc 渲成可点击链接;搜索引擎仍按 XML 解析
|
||
stylesheetIndex = `<?xml-stylesheet type="text/xsl" href="/static/sitemapindex.xsl"?>` + "\n"
|
||
stylesheetURLSet = `<?xml-stylesheet type="text/xsl" href="/static/sitemap.xsl"?>` + "\n"
|
||
|
||
// defaultMaxURLsPerFile 单文件 URL 上限;Google 上限 50000,留余量避免体积过大。
|
||
defaultMaxURLsPerFile = 40000
|
||
)
|
||
|
||
// URL 单个URL条目
|
||
type URL struct {
|
||
Loc string `xml:"loc"`
|
||
LastMod string `xml:"lastmod"`
|
||
ChangeFreq string `xml:"changefreq"` // always/hourly/daily/weekly/monthly/yearly/never
|
||
Priority float32 `xml:"priority"` // 0.0 - 1.0
|
||
}
|
||
|
||
// Sitemap 站点地图文件
|
||
type Sitemap struct {
|
||
XMLName xml.Name `xml:"urlset"`
|
||
Xmlns string `xml:"xmlns,attr"`
|
||
URLs []URL `xml:"url"`
|
||
}
|
||
|
||
// SitemapIndex 站点地图索引
|
||
type SitemapIndex struct {
|
||
XMLName xml.Name `xml:"sitemapindex"`
|
||
Xmlns string `xml:"xmlns,attr"`
|
||
Sitemaps []SitemapIndexEntry `xml:"sitemap"`
|
||
}
|
||
|
||
// SitemapIndexEntry 站点地图索引条目
|
||
type SitemapIndexEntry struct {
|
||
Loc string `xml:"loc"`
|
||
LastMod string `xml:"lastmod"`
|
||
}
|
||
|
||
// SitemapGenerator 站点地图生成器
|
||
type SitemapGenerator struct {
|
||
MaxURLsPerFile int // 每个sitemap文件最大URL数量
|
||
OutputDir string // 分片输出目录(/sitemaps)
|
||
IndexFile string // 索引文件路径;空则写到 OutputDir/sitemap.xml
|
||
WebAddr string // 网站地址
|
||
SitemapFiles []string // 生成的sitemap文件列表
|
||
}
|
||
|
||
// NewSitemapGenerator 创建新的站点地图生成器
|
||
func NewSitemapGenerator(webAddr, outputDir string) *SitemapGenerator {
|
||
return &SitemapGenerator{
|
||
MaxURLsPerFile: defaultMaxURLsPerFile,
|
||
OutputDir: outputDir,
|
||
WebAddr: webAddr,
|
||
SitemapFiles: make([]string, 0),
|
||
}
|
||
}
|
||
|
||
func (sg *SitemapGenerator) maxURLsPerFile() int {
|
||
if sg.MaxURLsPerFile > 0 {
|
||
return sg.MaxURLsPerFile
|
||
}
|
||
return defaultMaxURLsPerFile
|
||
}
|
||
|
||
// Generate 生成站点地图
|
||
func (sg *SitemapGenerator) Generate() error {
|
||
// 清空之前的文件列表
|
||
sg.SitemapFiles = make([]string, 0)
|
||
|
||
// 创建输出目录
|
||
if err := os.MkdirAll(sg.OutputDir, 0755); err != nil {
|
||
return err
|
||
}
|
||
deprecatedFiles := []string{
|
||
"sitemap-createpostlist.xml",
|
||
"sitemap-postlist.xml",
|
||
"sitemap-posttag.xml",
|
||
"sitemap-createvideolist.xml",
|
||
"sitemap-tag.xml",
|
||
"sitemap-translation.xml",
|
||
"sitemap-actress.xml",
|
||
"sitemap-videocategory.xml",
|
||
"sitemap-playletcategory.xml",
|
||
"sitemap-videotag.xml",
|
||
"sitemap-playlettag.xml",
|
||
"sitemap-archives.xml",
|
||
"sitemap-videos.xml",
|
||
}
|
||
for _, filename := range deprecatedFiles {
|
||
if err := os.Remove(filepath.Join(sg.OutputDir, filename)); err != nil && !os.IsNotExist(err) {
|
||
return err
|
||
}
|
||
}
|
||
sg.cleanupSitemapShards("sitemap-archives", 0)
|
||
sg.cleanupSitemapShards("sitemap-videos", 0)
|
||
if sg.IndexFile != "" && filepath.Clean(sg.IndexFile) != filepath.Clean(filepath.Join(sg.OutputDir, "sitemap.xml")) {
|
||
if err := os.Remove(filepath.Join(sg.OutputDir, "sitemap.xml")); err != nil && !os.IsNotExist(err) {
|
||
return err
|
||
}
|
||
}
|
||
getUrl(sg)
|
||
|
||
// 生成sitemap索引文件
|
||
return sg.generateSitemapIndex()
|
||
}
|
||
|
||
// writeSitemapFile 写入sitemap文件
|
||
func (sg *SitemapGenerator) writeSitemapFile(filename string, urls []URL) error {
|
||
sitemap := Sitemap{
|
||
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||
URLs: urls,
|
||
}
|
||
|
||
filePath := filepath.Join(sg.OutputDir, filename)
|
||
file, err := os.Create(filePath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer file.Close()
|
||
|
||
// 添加XML头
|
||
file.WriteString(xml.Header)
|
||
file.WriteString(stylesheetURLSet)
|
||
encoder := xml.NewEncoder(file)
|
||
encoder.Indent("", " ")
|
||
|
||
if err := encoder.Encode(sitemap); err != nil {
|
||
return err
|
||
}
|
||
|
||
// 记录生成的文件
|
||
sg.SitemapFiles = append(sg.SitemapFiles, filename)
|
||
return nil
|
||
}
|
||
|
||
// writeSitemapFiles 写入 sitemap 分片;超过上限时拆为 baseName-0.xml、baseName-1.xml ...
|
||
func (sg *SitemapGenerator) writeSitemapFiles(baseName string, urls []URL) error {
|
||
baseName = strings.TrimSuffix(strings.TrimSpace(baseName), ".xml")
|
||
if baseName == "" {
|
||
return nil
|
||
}
|
||
if len(urls) == 0 {
|
||
sg.cleanupSitemapShards(baseName, 0)
|
||
return nil
|
||
}
|
||
|
||
limit := sg.maxURLsPerFile()
|
||
shardCount := (len(urls) + limit - 1) / limit
|
||
if shardCount == 1 {
|
||
sg.cleanupSitemapShards(baseName, 1)
|
||
return sg.writeSitemapFile(baseName+".xml", urls)
|
||
}
|
||
|
||
sg.cleanupSitemapShards(baseName, shardCount)
|
||
for i := 0; i < shardCount; i++ {
|
||
start := i * limit
|
||
end := start + limit
|
||
if end > len(urls) {
|
||
end = len(urls)
|
||
}
|
||
filename := fmt.Sprintf("%s-%d.xml", baseName, i)
|
||
if err := sg.writeSitemapFile(filename, urls[start:end]); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// cleanupSitemapShards 清理旧分片:shardCount=1 保留 baseName.xml;>1 保留 baseName-0..N-1.xml。
|
||
func (sg *SitemapGenerator) cleanupSitemapShards(baseName string, shardCount int) {
|
||
entries, err := os.ReadDir(sg.OutputDir)
|
||
if err != nil {
|
||
return
|
||
}
|
||
prefix := baseName + "-"
|
||
for _, entry := range entries {
|
||
if entry.IsDir() {
|
||
continue
|
||
}
|
||
name := entry.Name()
|
||
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ".xml") {
|
||
idxText := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".xml")
|
||
idx, err := strconv.Atoi(idxText)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if shardCount <= 1 || idx >= shardCount {
|
||
_ = os.Remove(filepath.Join(sg.OutputDir, name))
|
||
}
|
||
continue
|
||
}
|
||
if shardCount > 1 && name == baseName+".xml" {
|
||
_ = os.Remove(filepath.Join(sg.OutputDir, name))
|
||
}
|
||
}
|
||
}
|
||
|
||
// generateSitemapIndex 生成sitemap索引文件
|
||
func (sg *SitemapGenerator) generateSitemapIndex() error {
|
||
index := SitemapIndex{
|
||
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||
Sitemaps: make([]SitemapIndexEntry, 0, len(sg.SitemapFiles)),
|
||
}
|
||
|
||
now := time.Now().Format(time.DateOnly)
|
||
webAddr := strings.TrimRight(strings.TrimSpace(sg.WebAddr), "/")
|
||
for _, filename := range sg.SitemapFiles {
|
||
index.Sitemaps = append(index.Sitemaps, SitemapIndexEntry{
|
||
Loc: fmt.Sprintf("%s/sitemaps/%s", webAddr, filename),
|
||
LastMod: now,
|
||
})
|
||
}
|
||
|
||
filePath := sg.IndexFile
|
||
if strings.TrimSpace(filePath) == "" {
|
||
filePath = filepath.Join(sg.OutputDir, "sitemap.xml")
|
||
}
|
||
if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
|
||
return err
|
||
}
|
||
file, err := os.Create(filePath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer file.Close()
|
||
|
||
// 添加XML头
|
||
file.WriteString(xml.Header)
|
||
file.WriteString(stylesheetIndex)
|
||
encoder := xml.NewEncoder(file)
|
||
encoder.Indent("", " ")
|
||
|
||
return encoder.Encode(index)
|
||
}
|