30 lines
803 B
Go
30 lines
803 B
Go
package seoser
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestBuildRobotsTXTUsesSingleCrawlerGroup(t *testing.T) {
|
|
body := BuildRobotsTXT("https://www.example.com")
|
|
if got := strings.Count(body, "User-agent:"); got != 1 {
|
|
t.Fatalf("User-agent group count = %d, want 1\n%s", got, body)
|
|
}
|
|
for _, want := range []string{
|
|
"Disallow: /api/",
|
|
"Disallow: /ui/",
|
|
"Disallow: /healthz",
|
|
"Sitemap: https://www.example.com/sitemap.xml",
|
|
"Host: www.example.com",
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
t.Fatalf("robots.txt missing %q\n%s", want, body)
|
|
}
|
|
}
|
|
for _, unwanted := range []string{"Disallow: /login", "Disallow: /search", "Disallow: /user-center"} {
|
|
if strings.Contains(body, unwanted) {
|
|
t.Fatalf("robots.txt should allow HTML page for noindex discovery: %q", unwanted)
|
|
}
|
|
}
|
|
}
|