Initial commit

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
fish03
2026-09-23 14:43:16 +07:00
co-authored by Claude Opus 5.5
commit bef9f8b4ed
663 changed files with 81305 additions and 0 deletions
+29
View File
@@ -0,0 +1,29 @@
package seoser
import (
"strings"
"testing"
)
func TestBuildRobotsTXTUsesSingleCrawlerGroup(t *testing.T) {
body := BuildRobotsTXT("https://www.example.com")
if got := strings.Count(body, "User-agent:"); got != 1 {
t.Fatalf("User-agent group count = %d, want 1\n%s", got, body)
}
for _, want := range []string{
"Disallow: /api/",
"Disallow: /ui/",
"Disallow: /healthz",
"Sitemap: https://www.example.com/sitemap.xml",
"Host: www.example.com",
} {
if !strings.Contains(body, want) {
t.Fatalf("robots.txt missing %q\n%s", want, body)
}
}
for _, unwanted := range []string{"Disallow: /login", "Disallow: /search", "Disallow: /user-center"} {
if strings.Contains(body, unwanted) {
t.Fatalf("robots.txt should allow HTML page for noindex discovery: %q", unwanted)
}
}
}