package sitemap import ( "encoding/xml" "fmt" "os" "path/filepath" "strconv" "strings" "time" ) const ( stylesheetIndex = `` + "\n" stylesheetURLSet = `` + "\n" defaultMaxURLsPerFile = 40000 // OutputDir 分片 sitemap 输出目录(相对 app 进程 cwd)。 // 对齐 jha-1045-hgdjnet-web:/server/web_91ys/h5/template/pages/sitemaps OutputDir = "../h5/template/pages/sitemaps" // PageOutputDir sitemap 索引输出目录,供 H5 /sitemap.xml 读取 PageOutputDir = "../h5/template/pages" ) type URL struct { Loc string `xml:"loc"` LastMod string `xml:"lastmod"` ChangeFreq string `xml:"changefreq"` Priority float32 `xml:"priority"` } type Sitemap struct { XMLName xml.Name `xml:"urlset"` Xmlns string `xml:"xmlns,attr"` URLs []URL `xml:"url"` } type SitemapIndex struct { XMLName xml.Name `xml:"sitemapindex"` Xmlns string `xml:"xmlns,attr"` Sitemaps []SitemapIndexEntry `xml:"sitemap"` } type SitemapIndexEntry struct { Loc string `xml:"loc"` LastMod string `xml:"lastmod"` } type SitemapGenerator struct { MaxURLsPerFile int OutputDir string WebAddr string SitemapFiles []string } func NewSitemapGenerator(webAddr, outputDir string) *SitemapGenerator { return &SitemapGenerator{ MaxURLsPerFile: defaultMaxURLsPerFile, OutputDir: outputDir, WebAddr: webAddr, SitemapFiles: make([]string, 0), } } func (sg *SitemapGenerator) maxURLsPerFile() int { if sg.MaxURLsPerFile > 0 { return sg.MaxURLsPerFile } return defaultMaxURLsPerFile } func (sg *SitemapGenerator) Generate() error { sg.SitemapFiles = make([]string, 0) if err := os.MkdirAll(sg.OutputDir, 0755); err != nil { return err } if err := os.MkdirAll(PageOutputDir, 0755); err != nil { return err } getURL(sg) if err := sg.generateSitemapIndex(); err != nil { return err } return copySitemapIndexToPageRoot() } func (sg *SitemapGenerator) writeSitemapFile(filename string, urls []URL) error { sitemap := Sitemap{ Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9", URLs: urls, } filePath := filepath.Join(sg.OutputDir, filename) file, err := os.Create(filePath) if err != nil { return err } defer file.Close() if _, err := file.WriteString(xml.Header); err != nil { return err } if _, err := file.WriteString(stylesheetURLSet); err != nil { return err } encoder := xml.NewEncoder(file) encoder.Indent("", " ") if err := encoder.Encode(sitemap); err != nil { return err } sg.SitemapFiles = append(sg.SitemapFiles, filename) return nil } func (sg *SitemapGenerator) writeSitemapFiles(baseName string, urls []URL) error { baseName = strings.TrimSuffix(strings.TrimSpace(baseName), ".xml") if baseName == "" { return nil } if len(urls) == 0 { sg.cleanupSitemapShards(baseName, 0) return nil } limit := sg.maxURLsPerFile() shardCount := (len(urls) + limit - 1) / limit if shardCount == 1 { sg.cleanupSitemapShards(baseName, 1) return sg.writeSitemapFile(baseName+".xml", urls) } sg.cleanupSitemapShards(baseName, shardCount) for i := 0; i < shardCount; i++ { start := i * limit end := start + limit if end > len(urls) { end = len(urls) } filename := fmt.Sprintf("%s-%d.xml", baseName, i) if err := sg.writeSitemapFile(filename, urls[start:end]); err != nil { return err } } return nil } func (sg *SitemapGenerator) cleanupSitemapShards(baseName string, shardCount int) { entries, err := os.ReadDir(sg.OutputDir) if err != nil { return } prefix := baseName + "-" for _, entry := range entries { if entry.IsDir() { continue } name := entry.Name() if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ".xml") { idxText := strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".xml") idx, err := strconv.Atoi(idxText) if err != nil { continue } if shardCount <= 1 || idx >= shardCount { _ = os.Remove(filepath.Join(sg.OutputDir, name)) } continue } if shardCount > 1 && name == baseName+".xml" { _ = os.Remove(filepath.Join(sg.OutputDir, name)) } } } func (sg *SitemapGenerator) generateSitemapIndex() error { index := SitemapIndex{ Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9", Sitemaps: make([]SitemapIndexEntry, 0, len(sg.SitemapFiles)), } now := time.Now().Format(time.DateOnly) for _, filename := range sg.SitemapFiles { index.Sitemaps = append(index.Sitemaps, SitemapIndexEntry{ Loc: fmt.Sprintf("%s/sitemaps/%s", strings.TrimSuffix(sg.WebAddr, "/"), filename), LastMod: now, }) } filePath := filepath.Join(sg.OutputDir, "sitemap.xml") file, err := os.Create(filePath) if err != nil { return err } defer file.Close() if _, err := file.WriteString(xml.Header); err != nil { return err } if _, err := file.WriteString(stylesheetIndex); err != nil { return err } encoder := xml.NewEncoder(file) encoder.Indent("", " ") return encoder.Encode(index) } func copySitemapIndexToPageRoot() error { src := filepath.Join(OutputDir, "sitemap.xml") dst := filepath.Join(PageOutputDir, "sitemap.xml") data, err := os.ReadFile(src) if err != nil { return err } return os.WriteFile(dst, data, 0644) } func writeRobotsTXT(webAddr string) error { body := buildRobotsTXT(webAddr) if body == "" { return nil } if err := os.MkdirAll(PageOutputDir, 0755); err != nil { return err } return os.WriteFile(filepath.Join(PageOutputDir, "robots.txt"), []byte(body), 0644) } func buildRobotsTXT(webAddr string) string { base := strings.TrimSuffix(strings.TrimSpace(webAddr), "/") if base == "" { return "" } body := "User-agent: *\nAllow: /\n\n" + "# 非 HTML 接口与播放器工具无需被搜索引擎抓取。\n" + "Disallow: /api/\n" + "Disallow: /h5/\n" + "Disallow: /ui/\n" + "Disallow: /healthz\n" + "Disallow: /enkey\n" + "Disallow: /newenkey\n" + "Disallow: /cdn-cgi/\n\n" + "# 搜索、登录和账户 HTML 保持可抓取,以便爬虫读取页面内 noindex。\n\n" + "Sitemap: " + base + "/sitemap.xml\n" if host := hostFromWebAddr(base); host != "" { body += "Host: " + host + "\n" } return body } func hostFromWebAddr(webAddr string) string { webAddr = strings.TrimSpace(webAddr) webAddr = strings.TrimPrefix(webAddr, "https://") webAddr = strings.TrimPrefix(webAddr, "http://") if i := strings.IndexByte(webAddr, '/'); i >= 0 { webAddr = webAddr[:i] } return webAddr }