Initial commit
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
package seoser
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BuildRobotsTXT 动态 robots.txt:优先读取 templates/page/robots.txt,并把 Sitemap 写成绝对地址。
|
||||
func BuildRobotsTXT(siteURL string) string {
|
||||
sitemap := AbsURL(siteURL, "/sitemap.xml")
|
||||
body := defaultRobotsTXT
|
||||
if raw, err := os.ReadFile("templates/page/robots.txt"); err == nil {
|
||||
if t := strings.TrimSpace(string(raw)); t != "" {
|
||||
body = t
|
||||
}
|
||||
}
|
||||
body = strings.ReplaceAll(body, "\r\n", "\n")
|
||||
body = sitemapLineRe.ReplaceAllString(body, "Sitemap: "+sitemap)
|
||||
if !strings.Contains(body, "Sitemap:") {
|
||||
body = strings.TrimRight(body, "\n") + "\n\nSitemap: " + sitemap + "\n"
|
||||
} else if !strings.HasSuffix(body, "\n") {
|
||||
body += "\n"
|
||||
}
|
||||
if host := hostFromSiteURL(siteURL); host != "" && !strings.Contains(body, "Host:") {
|
||||
body = strings.TrimRight(body, "\n") + "\nHost: " + host + "\n"
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func hostFromSiteURL(siteURL string) string {
|
||||
u, err := url.Parse(strings.TrimSpace(siteURL))
|
||||
if err != nil || u.Host == "" {
|
||||
return ""
|
||||
}
|
||||
return u.Host
|
||||
}
|
||||
|
||||
var sitemapLineRe = regexp.MustCompile(`(?m)^Sitemap:\s*\S+`)
|
||||
|
||||
const defaultRobotsTXT = `User-agent: *
|
||||
Allow: /
|
||||
|
||||
# 非 HTML 接口与工具路由不抓取
|
||||
Disallow: /api/
|
||||
Disallow: /ui/
|
||||
Disallow: /healthz
|
||||
Disallow: /h5/
|
||||
Disallow: /enkey
|
||||
Disallow: /newenkey
|
||||
|
||||
# 登录、搜索和账户 HTML 页面允许抓取,以便爬虫读取页面内 noindex。
|
||||
|
||||
Sitemap: /sitemap.xml
|
||||
`
|
||||
Reference in New Issue
Block a user