301 lines
7.5 KiB
Go
301 lines
7.5 KiB
Go
package sitemap
|
|
|
|
import (
|
|
"encoding/xml"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
stylesheetIndex = `<?xml-stylesheet type="text/xsl" href="/static/sitemapindex.xsl"?>` + "\n"
|
|
stylesheetURLSet = `<?xml-stylesheet type="text/xsl" href="/static/sitemap.xsl"?>` + "\n"
|
|
|
|
defaultMaxURLsPerFile = 40000
|
|
|
|
// OutputDir 分片 sitemap 输出目录（相对 app 进程 cwd）。
|
|
// 写在 H5 根目录的 sitemaps 下，不进前端 public；样式表由前端 /static/sitemap*.xsl 提供。
|
|
OutputDir = "../h5/sitemaps"
|
|
// PageOutputDir sitemap 索引与 robots.txt 输出目录，供 H5 /sitemap.xml 读取
|
|
PageOutputDir = "../h5"
|
|
)
|
|
|
|
type URL struct {
|
|
Loc string `xml:"loc"`
|
|
LastMod string `xml:"lastmod"`
|
|
ChangeFreq string `xml:"changefreq"`
|
|
Priority float32 `xml:"priority"`
|
|
}
|
|
|
|
type Sitemap struct {
|
|
XMLName xml.Name `xml:"urlset"`
|
|
Xmlns string `xml:"xmlns,attr"`
|
|
URLs []URL `xml:"url"`
|
|
}
|
|
|
|
type SitemapIndex struct {
|
|
XMLName xml.Name `xml:"sitemapindex"`
|
|
Xmlns string `xml:"xmlns,attr"`
|
|
Sitemaps []SitemapIndexEntry `xml:"sitemap"`
|
|
}
|
|
|
|
type SitemapIndexEntry struct {
|
|
Loc string `xml:"loc"`
|
|
LastMod string `xml:"lastmod"`
|
|
}
|
|
|
|
type SitemapGenerator struct {
|
|
MaxURLsPerFile int
|
|
OutputDir string
|
|
IndexFile string // 索引文件路径；空则写到 OutputDir/sitemap.xml
|
|
WebAddr string
|
|
SitemapFiles []string
|
|
}
|
|
|
|
func NewSitemapGenerator(webAddr, outputDir string) *SitemapGenerator {
|
|
return &SitemapGenerator{
|
|
MaxURLsPerFile: defaultMaxURLsPerFile,
|
|
OutputDir: outputDir,
|
|
WebAddr: webAddr,
|
|
SitemapFiles: make([]string, 0),
|
|
}
|
|
}
|
|
|
|
func (sg *SitemapGenerator) maxURLsPerFile() int {
|
|
if sg.MaxURLsPerFile > 0 {
|
|
return sg.MaxURLsPerFile
|
|
}
|
|
return defaultMaxURLsPerFile
|
|
}
|
|
|
|
func (sg *SitemapGenerator) Generate() error {
|
|
sg.SitemapFiles = make([]string, 0)
|
|
|
|
if err := os.MkdirAll(sg.OutputDir, 0755); err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(PageOutputDir, 0755); err != nil {
|
|
return err
|
|
}
|
|
|
|
getURL(sg)
|
|
|
|
if err := sg.generateSitemapIndex(); err != nil {
|
|
return err
|
|
}
|
|
if err := sg.publishSitemapIndex(); err != nil {
|
|
return err
|
|
}
|
|
return sg.removeShardIndexCopy()
|
|
}
|
|
|
|
// removeShardIndexCopy 索引写在 H5 根目录时，去掉分片目录里的同名索引，避免 /sitemaps/sitemap.xml 再被收录。
|
|
func (sg *SitemapGenerator) removeShardIndexCopy() error {
|
|
indexPath := strings.TrimSpace(sg.IndexFile)
|
|
if indexPath == "" {
|
|
return nil
|
|
}
|
|
shardIndex := filepath.Join(sg.OutputDir, "sitemap.xml")
|
|
if filepath.Clean(indexPath) == filepath.Clean(shardIndex) {
|
|
return nil
|
|
}
|
|
if err := os.Remove(shardIndex); err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (sg *SitemapGenerator) writeSitemapFile(filename string, urls []URL) error {
|
|
sitemap := Sitemap{
|
|
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
|
URLs: urls,
|
|
}
|
|
|
|
filePath := filepath.Join(sg.OutputDir, filename)
|
|
file, err := os.Create(filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
|
|
if _, err := file.WriteString(xml.Header); err != nil {
|
|
return err
|
|
}
|
|
if _, err := file.WriteString(stylesheetURLSet); err != nil {
|
|
return err
|
|
}
|
|
encoder := xml.NewEncoder(file)
|
|
encoder.Indent("", " ")
|
|
|
|
if err := encoder.Encode(sitemap); err != nil {
|
|
return err
|
|
}
|
|
|
|
sg.SitemapFiles = append(sg.SitemapFiles, filename)
|
|
return nil
|
|
}
|
|
|
|
func (sg *SitemapGenerator) writeSitemapFiles(baseName string, urls []URL) error {
|
|
baseName = strings.TrimSuffix(strings.TrimSpace(baseName), ".xml")
|
|
if baseName == "" {
|
|
return nil
|
|
}
|
|
if len(urls) == 0 {
|
|
sg.cleanupSitemapShards(baseName, 0)
|
|
return nil
|
|
}
|
|
|
|
limit := sg.maxURLsPerFile()
|
|
shardCount := (len(urls) + limit - 1) / limit
|
|
if shardCount == 1 {
|
|
sg.cleanupSitemapShards(baseName, 1)
|
|
return sg.writeSitemapFile(baseName+".xml", urls)
|
|
}
|
|
|
|
sg.cleanupSitemapShards(baseName, shardCount)
|
|
for i := 0; i < shardCount; i++ {
|
|
start := i * limit
|
|
end := start + limit
|
|
if end > len(urls) {
|
|
end = len(urls)
|
|
}
|
|
filename := fmt.Sprintf("%s-%d.xml", baseName, i)
|
|
if err := sg.writeSitemapFile(filename, urls[start:end]); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (sg *SitemapGenerator) cleanupSitemapShards(baseName string, shardCount int) {
|
|
entries, err := os.ReadDir(sg.OutputDir)
|
|
if err != nil {
|
|
return
|
|
}
|
|
prefix := baseName + "-"
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
name := entry.Name()
|
|
if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ".xml") {
|
|
idxText := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".xml")
|
|
idx, err := strconv.Atoi(idxText)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if shardCount <= 1 || idx >= shardCount {
|
|
_ = os.Remove(filepath.Join(sg.OutputDir, name))
|
|
}
|
|
continue
|
|
}
|
|
if shardCount > 1 && name == baseName+".xml" {
|
|
_ = os.Remove(filepath.Join(sg.OutputDir, name))
|
|
}
|
|
}
|
|
}
|
|
|
|
func (sg *SitemapGenerator) generateSitemapIndex() error {
|
|
index := SitemapIndex{
|
|
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
|
Sitemaps: make([]SitemapIndexEntry, 0, len(sg.SitemapFiles)),
|
|
}
|
|
|
|
now := time.Now().Format(time.DateOnly)
|
|
for _, filename := range sg.SitemapFiles {
|
|
index.Sitemaps = append(index.Sitemaps, SitemapIndexEntry{
|
|
Loc: fmt.Sprintf("%s/sitemaps/%s", strings.TrimSuffix(sg.WebAddr, "/"), filename),
|
|
LastMod: now,
|
|
})
|
|
}
|
|
|
|
filePath := strings.TrimSpace(sg.IndexFile)
|
|
if filePath == "" {
|
|
filePath = filepath.Join(sg.OutputDir, "sitemap.xml")
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
|
|
return err
|
|
}
|
|
file, err := os.Create(filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
|
|
if _, err := file.WriteString(xml.Header); err != nil {
|
|
return err
|
|
}
|
|
if _, err := file.WriteString(stylesheetIndex); err != nil {
|
|
return err
|
|
}
|
|
encoder := xml.NewEncoder(file)
|
|
encoder.Indent("", " ")
|
|
|
|
return encoder.Encode(index)
|
|
}
|
|
|
|
func (sg *SitemapGenerator) publishSitemapIndex() error {
|
|
src := strings.TrimSpace(sg.IndexFile)
|
|
if src == "" {
|
|
src = filepath.Join(sg.OutputDir, "sitemap.xml")
|
|
}
|
|
dst := filepath.Join(PageOutputDir, "sitemap.xml")
|
|
if filepath.Clean(src) == filepath.Clean(dst) {
|
|
return nil
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
data, err := os.ReadFile(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(dst, data, 0644)
|
|
}
|
|
|
|
func writeRobotsTXT(webAddr string) error {
|
|
body := buildRobotsTXT(webAddr)
|
|
if body == "" {
|
|
return nil
|
|
}
|
|
if err := os.MkdirAll(PageOutputDir, 0755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(filepath.Join(PageOutputDir, "robots.txt"), []byte(body), 0644)
|
|
}
|
|
|
|
func buildRobotsTXT(webAddr string) string {
|
|
base := strings.TrimSuffix(strings.TrimSpace(webAddr), "/")
|
|
if base == "" {
|
|
return ""
|
|
}
|
|
body := "User-agent: *\nAllow: /\n\n" +
|
|
"# 非 HTML 接口与播放器工具无需被搜索引擎抓取。\n" +
|
|
"Disallow: /api/\n" +
|
|
"Disallow: /h5/\n" +
|
|
"Disallow: /ui/\n" +
|
|
"Disallow: /healthz\n" +
|
|
"Disallow: /enkey\n" +
|
|
"Disallow: /newenkey\n" +
|
|
"Disallow: /cdn-cgi/\n\n" +
|
|
"# 搜索、登录和账户 HTML 保持可抓取，以便爬虫读取页面内 noindex。\n\n" +
|
|
"Sitemap: " + base + "/sitemap.xml\n"
|
|
if host := hostFromWebAddr(base); host != "" {
|
|
body += "Host: " + host + "\n"
|
|
}
|
|
return body
|
|
}
|
|
|
|
func hostFromWebAddr(webAddr string) string {
|
|
webAddr = strings.TrimSpace(webAddr)
|
|
webAddr = strings.TrimPrefix(webAddr, "https://")
|
|
webAddr = strings.TrimPrefix(webAddr, "http://")
|
|
if i := strings.IndexByte(webAddr, '/'); i >= 0 {
|
|
webAddr = webAddr[:i]
|
|
}
|
|
return webAddr
|
|
}
|