feat: implement a client for Thoth, the IP reputation database for Anubis (#637)

* feat(internal): add Thoth client and simple ASN checker Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thoth): cached ip to asn checker Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: go mod tidy Signed-off-by: Xe Iaso <me@xeiaso.net> * fix(thoth): minor testing fixups, ensure ASNChecker is Checker Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thoth): make ASNChecker instances Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thoth): add GeoIP checker Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thoth): store a thoth client in a context Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: refactor Checker type to its own package Signed-off-by: Xe Iaso <me@xeiaso.net> * test(thoth): add thoth mocking package, ignore context deadline exceeded errors Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thoth): pre-cache private ranges Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(lib/policy/config): enable thoth ASNs and GeoIP checker parsing Signed-off-by: Xe Iaso <me@xeiaso.net> * chore(thoth): refactor to move checker creation to the checker files Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(policy): enable thoth checks Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thothmock): test helper function for loading a mock thoth instance Signed-off-by: Xe Iaso <me@xeiaso.net> * feat: wire up Thoth, make thoth checks part of the default config Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: spelling Signed-off-by: Xe Iaso <me@xeiaso.net> * fix(thoth): mend staticcheck errors Signed-off-by: Xe Iaso <me@xeiaso.net> * docs(admin): add Thoth docs Signed-off-by: Xe Iaso <me@xeiaso.net> * chore(policy): update Thoth links in error messages Signed-off-by: Xe Iaso <me@xeiaso.net> * docs: update CHANGELOG Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: spelling Signed-off-by: Xe Iaso <me@xeiaso.net> * chore(docs/manifest): enable Thoth Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: add THOTH_INSECURE for contacting Thoth over plain TCP in extreme circumstances Signed-off-by: Xe Iaso <me@xeiaso.net> * test(thoth): use mock thoth when credentials aren't detected in the environment Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: spelling Signed-off-by: Xe Iaso <me@xeiaso.net> * fix(cmd/anubis): better warnings for half-configured Thoth setups Signed-off-by: Xe Iaso <me@xeiaso.net> * docs(botpolicies): link to Thoth geoip docs Signed-off-by: Xe Iaso <me@xeiaso.net> --------- Signed-off-by: Xe Iaso <me@xeiaso.net>
2025-06-16 11:57:32 -04:00 · 2025-06-16 11:57:32 -04:00 · e3826df3ab
commit e3826df3ab
parent 823d1be5d1
39 changed files with 1101 additions and 82 deletions
--- a/internal/thoth/cachediptoasn.go
+++ b/internal/thoth/cachediptoasn.go
@ -0,0 +1,84 @@
+package thoth
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/netip"
+
+	iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1"
+	"github.com/gaissmai/bart"
+	"google.golang.org/grpc"
+)
+
+type IPToASNWithCache struct {
+	next  iptoasnv1.IpToASNServiceClient
+	table *bart.Table[*iptoasnv1.LookupResponse]
+}
+
+func NewIpToASNWithCache(next iptoasnv1.IpToASNServiceClient) *IPToASNWithCache {
+	result := &IPToASNWithCache{
+		next:  next,
+		table: &bart.Table[*iptoasnv1.LookupResponse]{},
+	}
+
+	for _, pfx := range []netip.Prefix{
+		netip.MustParsePrefix("10.0.0.0/8"),         // RFC 1918
+		netip.MustParsePrefix("172.16.0.0/12"),      // RFC 1918
+		netip.MustParsePrefix("192.168.0.0/16"),     // RFC 1918
+		netip.MustParsePrefix("127.0.0.0/8"),        // Loopback
+		netip.MustParsePrefix("169.254.0.0/16"),     // Link-local
+		netip.MustParsePrefix("100.64.0.0/10"),      // CGNAT
+		netip.MustParsePrefix("192.0.0.0/24"),       // Protocol assignments
+		netip.MustParsePrefix("192.0.2.0/24"),       // TEST-NET-1
+		netip.MustParsePrefix("198.18.0.0/15"),      // Benchmarking
+		netip.MustParsePrefix("198.51.100.0/24"),    // TEST-NET-2
+		netip.MustParsePrefix("203.0.113.0/24"),     // TEST-NET-3
+		netip.MustParsePrefix("240.0.0.0/4"),        // Reserved
+		netip.MustParsePrefix("255.255.255.255/32"), // Broadcast
+		netip.MustParsePrefix("fc00::/7"),           // Unique local address
+		netip.MustParsePrefix("fe80::/10"),          // Link-local
+		netip.MustParsePrefix("::1/128"),            // Loopback
+		netip.MustParsePrefix("::/128"),             // Unspecified
+		netip.MustParsePrefix("100::/64"),           // Discard-only
+		netip.MustParsePrefix("2001:db8::/32"),      // Documentation
+	} {
+		result.table.Insert(pfx, &iptoasnv1.LookupResponse{Announced: false})
+	}
+
+	return result
+}
+
+func (ip2asn *IPToASNWithCache) Lookup(ctx context.Context, lr *iptoasnv1.LookupRequest, opts ...grpc.CallOption) (*iptoasnv1.LookupResponse, error) {
+	addr, err := netip.ParseAddr(lr.GetIpAddress())
+	if err != nil {
+		return nil, fmt.Errorf("input is not an IP address: %w", err)
+	}
+
+	cachedResponse, ok := ip2asn.table.Lookup(addr)
+	if ok {
+		return cachedResponse, nil
+	}
+
+	resp, err := ip2asn.next.Lookup(ctx, lr, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var errs []error
+	for _, cidr := range resp.GetCidr() {
+		pfx, err := netip.ParsePrefix(cidr)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		ip2asn.table.Insert(pfx, resp)
+	}
+
+	if len(errs) != 0 {
+		slog.Error("errors parsing IP prefixes", "err", errors.Join(errs...))
+	}
+
+	return resp, nil
+}