170 lines
3.8 KiB
Go
170 lines
3.8 KiB
Go
package turnstile
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
const forgejoRobots = `User-agent: *
|
|
Disallow: /api/
|
|
Disallow: /avatars/
|
|
Disallow: /user/
|
|
Disallow: /swagger.*.json
|
|
Disallow: /explore/*?*
|
|
|
|
Disallow: /repo/create
|
|
Disallow: /repo/migrate
|
|
Disallow: /org/create
|
|
Disallow: /*/*/fork
|
|
|
|
Disallow: /*/*/watchers
|
|
Disallow: /*/*/stargazers
|
|
Disallow: /*/*/forks
|
|
|
|
Disallow: /*/*/src/
|
|
Disallow: /*/*/blame/
|
|
Disallow: /*/*/commit/
|
|
Disallow: /*/*/commits/
|
|
Disallow: /*/*/raw/
|
|
Disallow: /*/*/media/
|
|
Disallow: /*/*/tags
|
|
Disallow: /*/*/graph
|
|
Disallow: /*/*/branches
|
|
Disallow: /*/*/compare
|
|
Disallow: /*/*/lastcommit/
|
|
Disallow: /*/*/rss/branch/
|
|
Disallow: /*/*/atom/branch/
|
|
|
|
Disallow: /*/*/activity
|
|
Disallow: /*/*/activity_author_data
|
|
|
|
Disallow: /*/*/actions
|
|
Disallow: /*/*/projects
|
|
Disallow: /*/*/labels
|
|
Disallow: /*/*/milestones
|
|
|
|
Disallow: /*/*/find/
|
|
Disallow: /*/*/tree-list/
|
|
Disallow: /*/*/search/
|
|
Disallow: /*/-/code
|
|
|
|
Disallow: /*/*/issues/new
|
|
Disallow: /*/*/pulls/*/files
|
|
Disallow: /*/*/pulls/*/commits
|
|
|
|
Disallow: /attachments/
|
|
Disallow: /*/*/attachments/
|
|
Disallow: /*/*/issues/*/attachments/
|
|
Disallow: /*/*/pulls/*/attachments/
|
|
Disallow: /*/*/releases/attachments
|
|
|
|
Disallow: /*/*/archive/
|
|
Disallow: /*.bundle$
|
|
Disallow: /*.patch$
|
|
Disallow: /*.diff$
|
|
Disallow: /*.atom$
|
|
Disallow: /*.rss$
|
|
|
|
Disallow: /*lang=*
|
|
Disallow: /*redirect_to=*
|
|
Disallow: /*tab=*
|
|
Disallow: /*q=*
|
|
Disallow: /*sort=*
|
|
Disallow: /*repo-search-archived=*
|
|
`
|
|
|
|
func TestParseAndMatchForgejo(t *testing.T) {
|
|
rules, err := parseRobotsStar(strings.NewReader(forgejoRobots))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(rules) == 0 {
|
|
t.Fatal("expected rules")
|
|
}
|
|
|
|
store := &robotsStore{rules: rules, loaded: true}
|
|
|
|
cases := []struct {
|
|
path, query string
|
|
want bool
|
|
}{
|
|
{"/owner/repo", "", false},
|
|
{"/owner/repo/src/main.go", "", true},
|
|
{"/owner/repo/blame/main.go", "", true},
|
|
{"/owner/repo/archive/main.zip", "", true},
|
|
{"/api/v1/repos", "", true},
|
|
{"/robots.txt", "", false},
|
|
{"/explore", "", false},
|
|
{"/explore", "q=foo", true},
|
|
{"/owner/repo", "q=test", true},
|
|
{"/something.bundle", "", true},
|
|
{"/something.bundle.backup", "", false},
|
|
{"/repo/create", "", true},
|
|
{"/owner/repo/issues/new", "", true},
|
|
{"/owner/repo/issues/1", "", false},
|
|
{"/swagger.v1.json", "", true},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
got := store.isDisallowed(tc.path, tc.query)
|
|
if got != tc.want {
|
|
t.Errorf("isDisallowed(%q, %q) = %v, want %v", tc.path, tc.query, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRobotsMatchBasics(t *testing.T) {
|
|
cases := []struct {
|
|
pattern, path string
|
|
want bool
|
|
}{
|
|
{"/api/", "/api/v1", true},
|
|
{"/api/", "/apiv1", false},
|
|
{"/*.bundle$", "/x.bundle", true},
|
|
{"/*.bundle$", "/x.bundle.z", false},
|
|
{"/*/*/src/", "/a/b/src/", true},
|
|
{"/*/*/src/", "/a/b/src/x", true},
|
|
{"/*/*/src/", "/a/b/tree/", false},
|
|
}
|
|
for _, tc := range cases {
|
|
if got := robotsMatch(tc.pattern, tc.path); got != tc.want {
|
|
t.Errorf("robotsMatch(%q, %q) = %v, want %v", tc.pattern, tc.path, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAllowOverridesShorterDisallow(t *testing.T) {
|
|
rules, err := parseRobotsStar(strings.NewReader(`User-agent: *
|
|
Disallow: /private
|
|
Allow: /private/public
|
|
`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store := &robotsStore{rules: rules, loaded: true}
|
|
if !store.isDisallowed("/private/secret", "") {
|
|
t.Fatal("expected /private/secret disallowed")
|
|
}
|
|
if store.isDisallowed("/private/public", "") {
|
|
t.Fatal("expected /private/public allowed via longer Allow")
|
|
}
|
|
}
|
|
|
|
func TestIgnoreNonStarGroups(t *testing.T) {
|
|
rules, err := parseRobotsStar(strings.NewReader(`User-agent: Googlebot
|
|
Disallow: /secret
|
|
|
|
User-agent: *
|
|
Disallow: /api/
|
|
`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store := &robotsStore{rules: rules, loaded: true}
|
|
if store.isDisallowed("/secret", "") {
|
|
t.Fatal("Googlebot-only rule should be ignored")
|
|
}
|
|
if !store.isDisallowed("/api/x", "") {
|
|
t.Fatal("star Disallow should apply")
|
|
}
|
|
}
|