fish03andClaude Opus 5.5
bef9f8b4ed
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
57 lines
1.4 KiB
Go
57 lines
1.4 KiB
Go
package seoser
|
|
|
|
import (
|
|
"net/url"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
)
|
|
|
|
// BuildRobotsTXT 动态 robots.txt：优先读取 templates/page/robots.txt，并把 Sitemap 写成绝对地址。
|
|
func BuildRobotsTXT(siteURL string) string {
|
|
sitemap := AbsURL(siteURL, "/sitemap.xml")
|
|
body := defaultRobotsTXT
|
|
if raw, err := os.ReadFile("templates/page/robots.txt"); err == nil {
|
|
if t := strings.TrimSpace(string(raw)); t != "" {
|
|
body = t
|
|
}
|
|
}
|
|
body = strings.ReplaceAll(body, "\r\n", "\n")
|
|
body = sitemapLineRe.ReplaceAllString(body, "Sitemap: "+sitemap)
|
|
if !strings.Contains(body, "Sitemap:") {
|
|
body = strings.TrimRight(body, "\n") + "\n\nSitemap: " + sitemap + "\n"
|
|
} else if !strings.HasSuffix(body, "\n") {
|
|
body += "\n"
|
|
}
|
|
if host := hostFromSiteURL(siteURL); host != "" && !strings.Contains(body, "Host:") {
|
|
body = strings.TrimRight(body, "\n") + "\nHost: " + host + "\n"
|
|
}
|
|
return body
|
|
}
|
|
|
|
func hostFromSiteURL(siteURL string) string {
|
|
u, err := url.Parse(strings.TrimSpace(siteURL))
|
|
if err != nil || u.Host == "" {
|
|
return ""
|
|
}
|
|
return u.Host
|
|
}
|
|
|
|
var sitemapLineRe = regexp.MustCompile(`(?m)^Sitemap:\s*\S+`)
|
|
|
|
const defaultRobotsTXT = `User-agent: *
|
|
Allow: /
|
|
|
|
# 非 HTML 接口与工具路由不抓取
|
|
Disallow: /api/
|
|
Disallow: /ui/
|
|
Disallow: /healthz
|
|
Disallow: /h5/
|
|
Disallow: /enkey
|
|
Disallow: /newenkey
|
|
|
|
# 登录、搜索和账户 HTML 页面允许抓取，以便爬虫读取页面内 noindex。
|
|
|
|
Sitemap: /sitemap.xml
|
|
`
|