turnstile-caddy/robots.go

package turnstile

import (
	"bufio"
	"fmt"
	"io"
	"net/http"
	"strings"
	"sync"
	"time"
)

// rule is one Allow or Disallow pattern from the User-agent: * group.
type rule struct {
	allow   bool
	pattern string
}

// robotsStore holds parsed Disallow/Allow rules for User-agent: *.
type robotsStore struct {
	mu       sync.RWMutex
	rules    []rule
	loaded   bool
	upstream string
	client   *http.Client
	refresh  time.Duration
	logf     func(string, ...any)
	stop     chan struct{}
	once     sync.Once
}

func newRobotsStore(upstream string, refresh time.Duration, logf func(string, ...any)) *robotsStore {
	if logf == nil {
		logf = func(string, ...any) {}
	}
	s := &robotsStore{
		upstream: upstream,
		client: &http.Client{
			Timeout: 15 * time.Second,
			// Do not follow redirects into challenged paths blindly.
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				if len(via) >= 5 {
					return fmt.Errorf("too many redirects")
				}
				return nil
			},
		},
		refresh: refresh,
		logf:    logf,
		stop:    make(chan struct{}),
	}
	go s.refreshLoop()
	_ = s.fetch()
	return s
}

func (s *robotsStore) close() {
	s.once.Do(func() { close(s.stop) })
}

// ensureLoaded returns whether rules are available. Fail-open until first successful fetch.
func (s *robotsStore) ensureLoaded() bool {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.loaded
}

func (s *robotsStore) refreshLoop() {
	ticker := time.NewTicker(s.refresh)
	defer ticker.Stop()
	for {
		select {
		case <-s.stop:
			return
		case <-ticker.C:
			if err := s.fetch(); err != nil {
				s.logf("robots.txt refresh failed: %v", err)
			}
		}
	}
}

func (s *robotsStore) fetch() error {
	s.mu.RLock()
	upstream := s.upstream
	s.mu.RUnlock()
	if upstream == "" {
		return fmt.Errorf("no upstream URL")
	}

	u := strings.TrimRight(upstream, "/") + "/robots.txt"
	req, err := http.NewRequest(http.MethodGet, u, nil)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", "turnstile-caddy/1.0")

	resp, err := s.client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected status %d from %s", resp.StatusCode, u)
	}

	rules, err := parseRobotsStar(resp.Body)
	if err != nil {
		return err
	}

	s.mu.Lock()
	s.rules = rules
	s.loaded = true
	s.mu.Unlock()
	s.logf("loaded %d rules from %s", len(rules), u)
	return nil
}

// isDisallowed reports whether path+query is disallowed for User-agent: *.
// Uses Google/RFC9309-style longest-match Allow/Disallow semantics.
func (s *robotsStore) isDisallowed(path, rawQuery string) bool {
	s.mu.RLock()
	defer s.mu.RUnlock()
	if !s.loaded {
		return false
	}

	target := path
	if rawQuery != "" {
		target = path + "?" + rawQuery
	}

	var bestLen int
	var bestAllow bool
	matched := false

	for _, r := range s.rules {
		if robotsMatch(r.pattern, target) {
			matched = true
			plen := len(r.pattern)
			if plen > bestLen {
				bestLen = plen
				bestAllow = r.allow
			}
		}
	}
	if !matched {
		return false
	}
	return !bestAllow
}

// parseRobotsStar extracts Allow/Disallow rules from the User-agent: * group.
func parseRobotsStar(r io.Reader) ([]rule, error) {
	scanner := bufio.NewScanner(r)
	scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)

	var (
		inStar bool
		rules  []rule
	)

	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if i := strings.Index(line, "#"); i >= 0 {
			line = strings.TrimSpace(line[:i])
		}
		if line == "" {
			continue
		}

		key, val, ok := splitRobotDirective(line)
		if !ok {
			continue
		}

		switch key {
		case "user-agent":
			ua := strings.ToLower(strings.TrimSpace(val))
			inStar = ua == "*"
		case "disallow":
			if !inStar || val == "" {
				continue
			}
			rules = append(rules, rule{allow: false, pattern: val})
		case "allow":
			if !inStar || val == "" {
				continue
			}
			rules = append(rules, rule{allow: true, pattern: val})
		}
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return rules, nil
}

func splitRobotDirective(line string) (key, val string, ok bool) {
	i := strings.IndexByte(line, ':')
	if i < 0 {
		return "", "", false
	}
	key = strings.ToLower(strings.TrimSpace(line[:i]))
	val = strings.TrimSpace(line[i+1:])
	return key, val, true
}

// robotsMatch implements robots.txt path pattern matching:
// - * matches any sequence
// - $ anchors end of string
// Patterns may match against path or path?query.
func robotsMatch(pattern, path string) bool {
	if pattern == "" {
		return false
	}

	endAnchor := false
	if strings.HasSuffix(pattern, "$") {
		endAnchor = true
		pattern = pattern[:len(pattern)-1]
	}

	return globMatch(pattern, path, endAnchor)
}

func globMatch(pattern, s string, endAnchor bool) bool {
	// Split pattern on * and require sequential matches.
	parts := strings.Split(pattern, "*")
	if len(parts) == 1 {
		if endAnchor {
			return s == pattern
		}
		return strings.HasPrefix(s, pattern)
	}

	// First part must be a prefix (unless pattern starts with *).
	if parts[0] != "" {
		if !strings.HasPrefix(s, parts[0]) {
			return false
		}
		s = s[len(parts[0]):]
	}

	for i := 1; i < len(parts)-1; i++ {
		p := parts[i]
		if p == "" {
			continue
		}
		idx := strings.Index(s, p)
		if idx < 0 {
			return false
		}
		s = s[idx+len(p):]
	}

	last := parts[len(parts)-1]
	if last == "" {
		// Pattern ends with *; if endAnchor was set with trailing $ after *,
		// empty last + endAnchor means match anything remaining including empty.
		return true
	}
	if endAnchor {
		return strings.HasSuffix(s, last)
	}
	// Remaining must contain last as a prefix of some suffix — i.e. Index ok,
	// and for non-anchored, presence anywhere after previous match is enough,
	// but Google treats the last segment as needing to appear; without $ the
	// trailing part is a required substring then anything may follow.
	idx := strings.Index(s, last)
	return idx >= 0
}