Initial commit: web_hghl_server
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,245 @@
|
||||
package sitemap
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// 浏览器用 XSLT 把 loc 渲成可点击链接;搜索引擎仍按 XML 解析
|
||||
stylesheetIndex = `<?xml-stylesheet type="text/xsl" href="/static/sitemapindex.xsl"?>` + "\n"
|
||||
stylesheetURLSet = `<?xml-stylesheet type="text/xsl" href="/static/sitemap.xsl"?>` + "\n"
|
||||
|
||||
// defaultMaxURLsPerFile 单文件 URL 上限;Google 上限 50000,留余量避免体积过大。
|
||||
defaultMaxURLsPerFile = 40000
|
||||
)
|
||||
|
||||
// URL 单个URL条目
|
||||
type URL struct {
|
||||
Loc string `xml:"loc"`
|
||||
LastMod string `xml:"lastmod"`
|
||||
ChangeFreq string `xml:"changefreq"` // always/hourly/daily/weekly/monthly/yearly/never
|
||||
Priority float32 `xml:"priority"` // 0.0 - 1.0
|
||||
}
|
||||
|
||||
// Sitemap 站点地图文件
|
||||
type Sitemap struct {
|
||||
XMLName xml.Name `xml:"urlset"`
|
||||
Xmlns string `xml:"xmlns,attr"`
|
||||
URLs []URL `xml:"url"`
|
||||
}
|
||||
|
||||
// SitemapIndex 站点地图索引
|
||||
type SitemapIndex struct {
|
||||
XMLName xml.Name `xml:"sitemapindex"`
|
||||
Xmlns string `xml:"xmlns,attr"`
|
||||
Sitemaps []SitemapIndexEntry `xml:"sitemap"`
|
||||
}
|
||||
|
||||
// SitemapIndexEntry 站点地图索引条目
|
||||
type SitemapIndexEntry struct {
|
||||
Loc string `xml:"loc"`
|
||||
LastMod string `xml:"lastmod"`
|
||||
}
|
||||
|
||||
// SitemapGenerator 站点地图生成器
|
||||
type SitemapGenerator struct {
|
||||
MaxURLsPerFile int // 每个sitemap文件最大URL数量
|
||||
OutputDir string // 分片输出目录(/sitemaps)
|
||||
IndexFile string // 索引文件路径;空则写到 OutputDir/sitemap.xml
|
||||
WebAddr string // 网站地址
|
||||
SitemapFiles []string // 生成的sitemap文件列表
|
||||
}
|
||||
|
||||
// NewSitemapGenerator 创建新的站点地图生成器
|
||||
func NewSitemapGenerator(webAddr, outputDir string) *SitemapGenerator {
|
||||
return &SitemapGenerator{
|
||||
MaxURLsPerFile: defaultMaxURLsPerFile,
|
||||
OutputDir: outputDir,
|
||||
WebAddr: webAddr,
|
||||
SitemapFiles: make([]string, 0),
|
||||
}
|
||||
}
|
||||
|
||||
func (sg *SitemapGenerator) maxURLsPerFile() int {
|
||||
if sg.MaxURLsPerFile > 0 {
|
||||
return sg.MaxURLsPerFile
|
||||
}
|
||||
return defaultMaxURLsPerFile
|
||||
}
|
||||
|
||||
// Generate 生成站点地图
|
||||
func (sg *SitemapGenerator) Generate() error {
|
||||
// 清空之前的文件列表
|
||||
sg.SitemapFiles = make([]string, 0)
|
||||
|
||||
// 创建输出目录
|
||||
if err := os.MkdirAll(sg.OutputDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
deprecatedFiles := []string{
|
||||
"sitemap-createpostlist.xml",
|
||||
"sitemap-postlist.xml",
|
||||
"sitemap-posttag.xml",
|
||||
"sitemap-createvideolist.xml",
|
||||
"sitemap-tag.xml",
|
||||
"sitemap-translation.xml",
|
||||
"sitemap-actress.xml",
|
||||
"sitemap-videocategory.xml",
|
||||
"sitemap-playletcategory.xml",
|
||||
"sitemap-videotag.xml",
|
||||
"sitemap-playlettag.xml",
|
||||
"sitemap-archives.xml",
|
||||
"sitemap-videos.xml",
|
||||
}
|
||||
for _, filename := range deprecatedFiles {
|
||||
if err := os.Remove(filepath.Join(sg.OutputDir, filename)); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
sg.cleanupSitemapShards("sitemap-archives", 0)
|
||||
sg.cleanupSitemapShards("sitemap-videos", 0)
|
||||
if sg.IndexFile != "" && filepath.Clean(sg.IndexFile) != filepath.Clean(filepath.Join(sg.OutputDir, "sitemap.xml")) {
|
||||
if err := os.Remove(filepath.Join(sg.OutputDir, "sitemap.xml")); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
getUrl(sg)
|
||||
|
||||
// 生成sitemap索引文件
|
||||
return sg.generateSitemapIndex()
|
||||
}
|
||||
|
||||
// writeSitemapFile 写入sitemap文件
|
||||
func (sg *SitemapGenerator) writeSitemapFile(filename string, urls []URL) error {
|
||||
sitemap := Sitemap{
|
||||
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||||
URLs: urls,
|
||||
}
|
||||
|
||||
filePath := filepath.Join(sg.OutputDir, filename)
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// 添加XML头
|
||||
file.WriteString(xml.Header)
|
||||
file.WriteString(stylesheetURLSet)
|
||||
encoder := xml.NewEncoder(file)
|
||||
encoder.Indent("", " ")
|
||||
|
||||
if err := encoder.Encode(sitemap); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 记录生成的文件
|
||||
sg.SitemapFiles = append(sg.SitemapFiles, filename)
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeSitemapFiles 写入 sitemap 分片;超过上限时拆为 baseName-0.xml、baseName-1.xml ...
|
||||
func (sg *SitemapGenerator) writeSitemapFiles(baseName string, urls []URL) error {
|
||||
baseName = strings.TrimSuffix(strings.TrimSpace(baseName), ".xml")
|
||||
if baseName == "" {
|
||||
return nil
|
||||
}
|
||||
if len(urls) == 0 {
|
||||
sg.cleanupSitemapShards(baseName, 0)
|
||||
return nil
|
||||
}
|
||||
|
||||
limit := sg.maxURLsPerFile()
|
||||
shardCount := (len(urls) + limit - 1) / limit
|
||||
if shardCount == 1 {
|
||||
sg.cleanupSitemapShards(baseName, 1)
|
||||
return sg.writeSitemapFile(baseName+".xml", urls)
|
||||
}
|
||||
|
||||
sg.cleanupSitemapShards(baseName, shardCount)
|
||||
for i := 0; i < shardCount; i++ {
|
||||
start := i * limit
|
||||
end := start + limit
|
||||
if end > len(urls) {
|
||||
end = len(urls)
|
||||
}
|
||||
filename := fmt.Sprintf("%s-%d.xml", baseName, i)
|
||||
if err := sg.writeSitemapFile(filename, urls[start:end]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupSitemapShards 清理旧分片:shardCount=1 保留 baseName.xml;>1 保留 baseName-0..N-1.xml。
|
||||
func (sg *SitemapGenerator) cleanupSitemapShards(baseName string, shardCount int) {
|
||||
entries, err := os.ReadDir(sg.OutputDir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
prefix := baseName + "-"
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ".xml") {
|
||||
idxText := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".xml")
|
||||
idx, err := strconv.Atoi(idxText)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if shardCount <= 1 || idx >= shardCount {
|
||||
_ = os.Remove(filepath.Join(sg.OutputDir, name))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if shardCount > 1 && name == baseName+".xml" {
|
||||
_ = os.Remove(filepath.Join(sg.OutputDir, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// generateSitemapIndex 生成sitemap索引文件
|
||||
func (sg *SitemapGenerator) generateSitemapIndex() error {
|
||||
index := SitemapIndex{
|
||||
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||||
Sitemaps: make([]SitemapIndexEntry, 0, len(sg.SitemapFiles)),
|
||||
}
|
||||
|
||||
now := time.Now().Format(time.DateOnly)
|
||||
webAddr := strings.TrimRight(strings.TrimSpace(sg.WebAddr), "/")
|
||||
for _, filename := range sg.SitemapFiles {
|
||||
index.Sitemaps = append(index.Sitemaps, SitemapIndexEntry{
|
||||
Loc: fmt.Sprintf("%s/sitemaps/%s", webAddr, filename),
|
||||
LastMod: now,
|
||||
})
|
||||
}
|
||||
|
||||
filePath := sg.IndexFile
|
||||
if strings.TrimSpace(filePath) == "" {
|
||||
filePath = filepath.Join(sg.OutputDir, "sitemap.xml")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// 添加XML头
|
||||
file.WriteString(xml.Header)
|
||||
file.WriteString(stylesheetIndex)
|
||||
encoder := xml.NewEncoder(file)
|
||||
encoder.Indent("", " ")
|
||||
|
||||
return encoder.Encode(index)
|
||||
}
|
||||
Reference in New Issue
Block a user