package seoser import ( "strings" "testing" ) func TestBuildRobotsTXTUsesSingleCrawlerGroup(t *testing.T) { body := BuildRobotsTXT("https://www.example.com") if got := strings.Count(body, "User-agent:"); got != 1 { t.Fatalf("User-agent group count = %d, want 1\n%s", got, body) } for _, want := range []string{ "Disallow: /api/", "Disallow: /ui/", "Disallow: /healthz", "Sitemap: https://www.example.com/sitemap.xml", "Host: www.example.com", } { if !strings.Contains(body, want) { t.Fatalf("robots.txt missing %q\n%s", want, body) } } for _, unwanted := range []string{"Disallow: /login", "Disallow: /search", "Disallow: /user-center"} { if strings.Contains(body, unwanted) { t.Fatalf("robots.txt should allow HTML page for noindex discovery: %q", unwanted) } } }