@@ -0,0 +1,29 @@
|
||||
package seoser
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildRobotsTXTUsesSingleCrawlerGroup(t *testing.T) {
|
||||
body := BuildRobotsTXT("https://www.example.com")
|
||||
if got := strings.Count(body, "User-agent:"); got != 1 {
|
||||
t.Fatalf("User-agent group count = %d, want 1\n%s", got, body)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"Disallow: /api/",
|
||||
"Disallow: /ui/",
|
||||
"Disallow: /healthz",
|
||||
"Sitemap: https://www.example.com/sitemap.xml",
|
||||
"Host: www.example.com",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Fatalf("robots.txt missing %q\n%s", want, body)
|
||||
}
|
||||
}
|
||||
for _, unwanted := range []string{"Disallow: /login", "Disallow: /search", "Disallow: /user-center"} {
|
||||
if strings.Contains(body, unwanted) {
|
||||
t.Fatalf("robots.txt should allow HTML page for noindex discovery: %q", unwanted)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user