package turnstile import ( "bufio" "fmt" "io" "net/http" "strings" "sync" "time" ) // rule is one Allow or Disallow pattern from the User-agent: * group. type rule struct { allow bool pattern string } // robotsStore holds parsed Disallow/Allow rules for User-agent: *. type robotsStore struct { mu sync.RWMutex rules []rule loaded bool upstream string client *http.Client refresh time.Duration logf func(string, ...any) stop chan struct{} once sync.Once } func newRobotsStore(upstream string, refresh time.Duration, logf func(string, ...any)) *robotsStore { if logf == nil { logf = func(string, ...any) {} } s := &robotsStore{ upstream: upstream, client: &http.Client{ Timeout: 15 * time.Second, // Do not follow redirects into challenged paths blindly. CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= 5 { return fmt.Errorf("too many redirects") } return nil }, }, refresh: refresh, logf: logf, stop: make(chan struct{}), } go s.refreshLoop() _ = s.fetch() return s } func (s *robotsStore) close() { s.once.Do(func() { close(s.stop) }) } // ensureLoaded returns whether rules are available. Fail-open until first successful fetch. func (s *robotsStore) ensureLoaded() bool { s.mu.RLock() defer s.mu.RUnlock() return s.loaded } func (s *robotsStore) refreshLoop() { ticker := time.NewTicker(s.refresh) defer ticker.Stop() for { select { case <-s.stop: return case <-ticker.C: if err := s.fetch(); err != nil { s.logf("robots.txt refresh failed: %v", err) } } } } func (s *robotsStore) fetch() error { s.mu.RLock() upstream := s.upstream s.mu.RUnlock() if upstream == "" { return fmt.Errorf("no upstream URL") } u := strings.TrimRight(upstream, "/") + "/robots.txt" req, err := http.NewRequest(http.MethodGet, u, nil) if err != nil { return err } req.Header.Set("User-Agent", "turnstile-caddy/1.0") resp, err := s.client.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status %d from %s", resp.StatusCode, u) } rules, err := parseRobotsStar(resp.Body) if err != nil { return err } s.mu.Lock() s.rules = rules s.loaded = true s.mu.Unlock() s.logf("loaded %d rules from %s", len(rules), u) return nil } // isDisallowed reports whether path+query is disallowed for User-agent: *. // Uses Google/RFC9309-style longest-match Allow/Disallow semantics. func (s *robotsStore) isDisallowed(path, rawQuery string) bool { s.mu.RLock() defer s.mu.RUnlock() if !s.loaded { return false } target := path if rawQuery != "" { target = path + "?" + rawQuery } var bestLen int var bestAllow bool matched := false for _, r := range s.rules { if robotsMatch(r.pattern, target) { matched = true plen := len(r.pattern) if plen > bestLen { bestLen = plen bestAllow = r.allow } } } if !matched { return false } return !bestAllow } // parseRobotsStar extracts Allow/Disallow rules from the User-agent: * group. func parseRobotsStar(r io.Reader) ([]rule, error) { scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) var ( inStar bool rules []rule ) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if i := strings.Index(line, "#"); i >= 0 { line = strings.TrimSpace(line[:i]) } if line == "" { continue } key, val, ok := splitRobotDirective(line) if !ok { continue } switch key { case "user-agent": ua := strings.ToLower(strings.TrimSpace(val)) inStar = ua == "*" case "disallow": if !inStar || val == "" { continue } rules = append(rules, rule{allow: false, pattern: val}) case "allow": if !inStar || val == "" { continue } rules = append(rules, rule{allow: true, pattern: val}) } } if err := scanner.Err(); err != nil { return nil, err } return rules, nil } func splitRobotDirective(line string) (key, val string, ok bool) { i := strings.IndexByte(line, ':') if i < 0 { return "", "", false } key = strings.ToLower(strings.TrimSpace(line[:i])) val = strings.TrimSpace(line[i+1:]) return key, val, true } // robotsMatch implements robots.txt path pattern matching: // - * matches any sequence // - $ anchors end of string // Patterns may match against path or path?query. func robotsMatch(pattern, path string) bool { if pattern == "" { return false } endAnchor := false if strings.HasSuffix(pattern, "$") { endAnchor = true pattern = pattern[:len(pattern)-1] } return globMatch(pattern, path, endAnchor) } func globMatch(pattern, s string, endAnchor bool) bool { // Split pattern on * and require sequential matches. parts := strings.Split(pattern, "*") if len(parts) == 1 { if endAnchor { return s == pattern } return strings.HasPrefix(s, pattern) } // First part must be a prefix (unless pattern starts with *). if parts[0] != "" { if !strings.HasPrefix(s, parts[0]) { return false } s = s[len(parts[0]):] } for i := 1; i < len(parts)-1; i++ { p := parts[i] if p == "" { continue } idx := strings.Index(s, p) if idx < 0 { return false } s = s[idx+len(p):] } last := parts[len(parts)-1] if last == "" { // Pattern ends with *; if endAnchor was set with trailing $ after *, // empty last + endAnchor means match anything remaining including empty. return true } if endAnchor { return strings.HasSuffix(s, last) } // Remaining must contain last as a prefix of some suffix — i.e. Index ok, // and for non-anchored, presence anywhere after previous match is enough, // but Google treats the last segment as needing to appear; without $ the // trailing part is a required substring then anything may follow. idx := strings.Index(s, last) return idx >= 0 }