package seoser import ( "net/url" "os" "regexp" "strings" templateassets "videoweb/app/template" ) // BuildRobotsTXT 动态 robots.txt:优先读取当前项目模板文件,并把 Sitemap 写成绝对地址。 func BuildRobotsTXT(siteURL string) string { sitemap := AbsURL(siteURL, "/sitemap.xml") body := defaultRobotsTXT loadedFromDisk := false for _, path := range []string{ "template/pages/robots.txt", "templates/pages/robots.txt", "app/template/pages/robots.txt", "app/templates/pages/robots.txt", } { if raw, err := os.ReadFile(path); err == nil { if t := strings.TrimSpace(string(raw)); t != "" { body = t loadedFromDisk = true break } } } if !loadedFromDisk { if raw, err := templateassets.FS.ReadFile("pages/robots.txt"); err == nil { if t := strings.TrimSpace(string(raw)); t != "" { body = t } } } body = strings.ReplaceAll(body, "\r\n", "\n") body = sitemapLineRe.ReplaceAllString(body, "Sitemap: "+sitemap) if !strings.Contains(body, "Sitemap:") { body = strings.TrimRight(body, "\n") + "\n\nSitemap: " + sitemap + "\n" } else if !strings.HasSuffix(body, "\n") { body += "\n" } if host := hostFromSiteURL(siteURL); host != "" && !strings.Contains(body, "Host:") { body = strings.TrimRight(body, "\n") + "\nHost: " + host + "\n" } return body } func hostFromSiteURL(siteURL string) string { u, err := url.Parse(strings.TrimSpace(siteURL)) if err != nil || u.Host == "" { return "" } return u.Host } var sitemapLineRe = regexp.MustCompile(`(?m)^Sitemap:\s*\S+`) const defaultRobotsTXT = `User-agent: * Allow: / # 非 HTML 接口与工具路由不抓取 Disallow: /api/ Disallow: /ui/ Disallow: /healthz Disallow: /cdn-cgi/ Disallow: /h5/ Disallow: /enkey Disallow: /newenkey # 登录、搜索和账户 HTML 页面允许抓取,以便爬虫读取页面内 noindex。 Sitemap: /sitemap.xml `