package turnstile import ( "strings" "testing" ) const forgejoRobots = `User-agent: * Disallow: /api/ Disallow: /avatars/ Disallow: /user/ Disallow: /swagger.*.json Disallow: /explore/*?* Disallow: /repo/create Disallow: /repo/migrate Disallow: /org/create Disallow: /*/*/fork Disallow: /*/*/watchers Disallow: /*/*/stargazers Disallow: /*/*/forks Disallow: /*/*/src/ Disallow: /*/*/blame/ Disallow: /*/*/commit/ Disallow: /*/*/commits/ Disallow: /*/*/raw/ Disallow: /*/*/media/ Disallow: /*/*/tags Disallow: /*/*/graph Disallow: /*/*/branches Disallow: /*/*/compare Disallow: /*/*/lastcommit/ Disallow: /*/*/rss/branch/ Disallow: /*/*/atom/branch/ Disallow: /*/*/activity Disallow: /*/*/activity_author_data Disallow: /*/*/actions Disallow: /*/*/projects Disallow: /*/*/labels Disallow: /*/*/milestones Disallow: /*/*/find/ Disallow: /*/*/tree-list/ Disallow: /*/*/search/ Disallow: /*/-/code Disallow: /*/*/issues/new Disallow: /*/*/pulls/*/files Disallow: /*/*/pulls/*/commits Disallow: /attachments/ Disallow: /*/*/attachments/ Disallow: /*/*/issues/*/attachments/ Disallow: /*/*/pulls/*/attachments/ Disallow: /*/*/releases/attachments Disallow: /*/*/archive/ Disallow: /*.bundle$ Disallow: /*.patch$ Disallow: /*.diff$ Disallow: /*.atom$ Disallow: /*.rss$ Disallow: /*lang=* Disallow: /*redirect_to=* Disallow: /*tab=* Disallow: /*q=* Disallow: /*sort=* Disallow: /*repo-search-archived=* ` func TestParseAndMatchForgejo(t *testing.T) { rules, err := parseRobotsStar(strings.NewReader(forgejoRobots)) if err != nil { t.Fatal(err) } if len(rules) == 0 { t.Fatal("expected rules") } store := &robotsStore{rules: rules, loaded: true} cases := []struct { path, query string want bool }{ {"/owner/repo", "", false}, {"/owner/repo/src/main.go", "", true}, {"/owner/repo/blame/main.go", "", true}, {"/owner/repo/archive/main.zip", "", true}, {"/api/v1/repos", "", true}, {"/robots.txt", "", false}, {"/explore", "", false}, {"/explore", "q=foo", true}, {"/owner/repo", "q=test", true}, {"/something.bundle", "", true}, {"/something.bundle.backup", "", false}, {"/repo/create", "", true}, {"/owner/repo/issues/new", "", true}, {"/owner/repo/issues/1", "", false}, {"/swagger.v1.json", "", true}, } for _, tc := range cases { got := store.isDisallowed(tc.path, tc.query) if got != tc.want { t.Errorf("isDisallowed(%q, %q) = %v, want %v", tc.path, tc.query, got, tc.want) } } } func TestRobotsMatchBasics(t *testing.T) { cases := []struct { pattern, path string want bool }{ {"/api/", "/api/v1", true}, {"/api/", "/apiv1", false}, {"/*.bundle$", "/x.bundle", true}, {"/*.bundle$", "/x.bundle.z", false}, {"/*/*/src/", "/a/b/src/", true}, {"/*/*/src/", "/a/b/src/x", true}, {"/*/*/src/", "/a/b/tree/", false}, } for _, tc := range cases { if got := robotsMatch(tc.pattern, tc.path); got != tc.want { t.Errorf("robotsMatch(%q, %q) = %v, want %v", tc.pattern, tc.path, got, tc.want) } } } func TestAllowOverridesShorterDisallow(t *testing.T) { rules, err := parseRobotsStar(strings.NewReader(`User-agent: * Disallow: /private Allow: /private/public `)) if err != nil { t.Fatal(err) } store := &robotsStore{rules: rules, loaded: true} if !store.isDisallowed("/private/secret", "") { t.Fatal("expected /private/secret disallowed") } if store.isDisallowed("/private/public", "") { t.Fatal("expected /private/public allowed via longer Allow") } } func TestIgnoreNonStarGroups(t *testing.T) { rules, err := parseRobotsStar(strings.NewReader(`User-agent: Googlebot Disallow: /secret User-agent: * Disallow: /api/ `)) if err != nil { t.Fatal(err) } store := &robotsStore{rules: rules, loaded: true} if store.isDisallowed("/secret", "") { t.Fatal("Googlebot-only rule should be ignored") } if !store.isDisallowed("/api/x", "") { t.Fatal("star Disallow should apply") } }