feat: implement a client for Thoth, the IP reputation database for Anubis (#637)

* feat(internal): add Thoth client and simple ASN checker

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thoth): cached ip to asn checker

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: go mod tidy

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(thoth): minor testing fixups, ensure ASNChecker is Checker

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thoth): make ASNChecker instances

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thoth): add GeoIP checker

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thoth): store a thoth client in a context

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: refactor Checker type to its own package

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(thoth): add thoth mocking package, ignore context deadline exceeded errors

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thoth): pre-cache private ranges

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(lib/policy/config): enable thoth ASNs and GeoIP checker parsing

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(thoth): refactor to move checker creation to the checker files

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(policy): enable thoth checks

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thothmock): test helper function for loading a mock thoth instance

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat: wire up Thoth, make thoth checks part of the default config

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(thoth): mend staticcheck errors

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs(admin): add Thoth docs

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(policy): update Thoth links in error messages

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs: update CHANGELOG

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(docs/manifest): enable Thoth

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: add THOTH_INSECURE for contacting Thoth over plain TCP in extreme circumstances

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(thoth): use mock thoth when credentials aren't detected in the environment

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(cmd/anubis): better warnings for half-configured Thoth setups

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs(botpolicies): link to Thoth geoip docs

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>

lib/policy/config/asn.go

@@ -0,0 +1,44 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	ErrPrivateASN = errors.New("bot.ASNs: you have specified a private use ASN")
+)
+
+type ASNs struct {
+	Match []uint32 `json:"match"`
+}
+
+func (a *ASNs) Valid() error {
+	var errs []error
+
+	for _, asn := range a.Match {
+		if isPrivateASN(asn) {
+			errs = append(errs, fmt.Errorf("%w: %d is private (see RFC 6996)", ErrPrivateASN, asn))
+		}
+	}
+
+	if len(errs) != 0 {
+		return fmt.Errorf("bot.ASNs: invalid ASN settings: %w", errors.Join(errs...))
+	}
+
+	return nil
+}
+
+// isPrivateASN checks if an ASN is in the private use area.
+//
+// Based on RFC 6996 and IANA allocations.
+func isPrivateASN(asn uint32) bool {
+	switch {
+	case asn >= 64512 && asn <= 65534:
+		return true
+	case asn >= 4200000000 && asn <= 4294967294:
+		return true
+	default:
+		return false
+	}
+}

lib/policy/config/config.go

@@ -55,6 +55,10 @@ type BotConfig struct {
 	Name           string            `json:"name" yaml:"name"`
 	Action         Rule              `json:"action" yaml:"action"`
 	RemoteAddr     []string          `json:"remote_addresses,omitempty" yaml:"remote_addresses,omitempty"`
+
+	// Thoth features
+	GeoIP *GeoIP `json:"geoip,omitempty"`
+	ASNs  *ASNs  `json:"asns,omitempty"`
 }
 
 func (b BotConfig) Zero() bool {
@@ -66,6 +70,8 @@ func (b BotConfig) Zero() bool {
 		b.Action != "",
 		len(b.RemoteAddr) != 0,
 		b.Challenge != nil,
+		b.GeoIP != nil,
+		b.ASNs != nil,
 	} {
 		if cond {
 			return false
@@ -85,7 +91,9 @@ func (b *BotConfig) Valid() error {
 	allFieldsEmpty := b.UserAgentRegex == nil &&
 		b.PathRegex == nil &&
 		len(b.RemoteAddr) == 0 &&
-		len(b.HeadersRegex) == 0
+		len(b.HeadersRegex) == 0 &&
+		b.ASNs == nil &&
+		b.GeoIP == nil
 
 	if allFieldsEmpty && b.Expression == nil {
 		errs = append(errs, ErrBotMustHaveUserAgentOrPath)

lib/policy/config/geoip.go

@@ -0,0 +1,36 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var (
+	countryCodeRegexp = regexp.MustCompile(`^\w{2}$`)
+
+	ErrNotCountryCode = errors.New("config.Bot: invalid country code")
+)
+
+type GeoIP struct {
+	Countries []string `json:"countries"`
+}
+
+func (g *GeoIP) Valid() error {
+	var errs []error
+
+	for i, cc := range g.Countries {
+		if !countryCodeRegexp.MatchString(cc) {
+			errs = append(errs, fmt.Errorf("%w: %s", ErrNotCountryCode, cc))
+		}
+
+		g.Countries[i] = strings.ToLower(cc)
+	}
+
+	if len(errs) != 0 {
+		return fmt.Errorf("bot.GeoIP: invalid GeoIP settings: %w", errors.Join(errs...))
+	}
+
+	return nil
+}

lib/policy/config/testdata/good/challenge_cloudflare.yaml

@@ -0,0 +1,6 @@
+bots:
+  - name: challenge-cloudflare
+    action: CHALLENGE
+    asns:
+      match:
+        - 13335 # Cloudflare

lib/policy/config/testdata/good/geoip_us.yaml

@@ -0,0 +1,6 @@
+bots:
+  - name: compute-tarrif-us
+    action: CHALLENGE
+    geoip:
+      countries:
+        - US