feat: implement a client for Thoth, the IP reputation database for Anubis (#637)
* feat(internal): add Thoth client and simple ASN checker Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thoth): cached ip to asn checker Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: go mod tidy Signed-off-by: Xe Iaso <me@xeiaso.net> * fix(thoth): minor testing fixups, ensure ASNChecker is Checker Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thoth): make ASNChecker instances Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thoth): add GeoIP checker Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thoth): store a thoth client in a context Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: refactor Checker type to its own package Signed-off-by: Xe Iaso <me@xeiaso.net> * test(thoth): add thoth mocking package, ignore context deadline exceeded errors Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thoth): pre-cache private ranges Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(lib/policy/config): enable thoth ASNs and GeoIP checker parsing Signed-off-by: Xe Iaso <me@xeiaso.net> * chore(thoth): refactor to move checker creation to the checker files Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(policy): enable thoth checks Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(thothmock): test helper function for loading a mock thoth instance Signed-off-by: Xe Iaso <me@xeiaso.net> * feat: wire up Thoth, make thoth checks part of the default config Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: spelling Signed-off-by: Xe Iaso <me@xeiaso.net> * fix(thoth): mend staticcheck errors Signed-off-by: Xe Iaso <me@xeiaso.net> * docs(admin): add Thoth docs Signed-off-by: Xe Iaso <me@xeiaso.net> * chore(policy): update Thoth links in error messages Signed-off-by: Xe Iaso <me@xeiaso.net> * docs: update CHANGELOG Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: spelling Signed-off-by: Xe Iaso <me@xeiaso.net> * chore(docs/manifest): enable Thoth Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: add THOTH_INSECURE for contacting Thoth over plain TCP in extreme circumstances Signed-off-by: Xe Iaso <me@xeiaso.net> * test(thoth): use mock thoth when credentials aren't detected in the environment Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: spelling Signed-off-by: Xe Iaso <me@xeiaso.net> * fix(cmd/anubis): better warnings for half-configured Thoth setups Signed-off-by: Xe Iaso <me@xeiaso.net> * docs(botpolicies): link to Thoth geoip docs Signed-off-by: Xe Iaso <me@xeiaso.net> --------- Signed-off-by: Xe Iaso <me@xeiaso.net>
This commit is contained in:
Xe Iaso
GitHub
parent
823d1be5d1
commit
e3826df3ab
39 changed files with 1101 additions and 82 deletions
|
|
@ -4,11 +4,12 @@ import (
|
|||
"fmt"
|
||||
|
||||
"github.com/TecharoHQ/anubis/internal"
|
||||
"github.com/TecharoHQ/anubis/lib/policy/checker"
|
||||
"github.com/TecharoHQ/anubis/lib/policy/config"
|
||||
)
|
||||
|
||||
type Bot struct {
|
||||
Rules Checker
|
||||
Rules checker.Impl
|
||||
Challenge *config.ChallengeRules
|
||||
Weight *config.Weight
|
||||
Name string
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/TecharoHQ/anubis/internal"
|
||||
"github.com/TecharoHQ/anubis/lib/policy/checker"
|
||||
"github.com/yl2chen/cidranger"
|
||||
)
|
||||
|
||||
|
|
@ -16,37 +17,6 @@ var (
|
|||
ErrMisconfiguration = errors.New("[unexpected] policy: administrator misconfiguration")
|
||||
)
|
||||
|
||||
type Checker interface {
|
||||
Check(*http.Request) (bool, error)
|
||||
Hash() string
|
||||
}
|
||||
|
||||
type CheckerList []Checker
|
||||
|
||||
func (cl CheckerList) Check(r *http.Request) (bool, error) {
|
||||
for _, c := range cl {
|
||||
ok, err := c.Check(r)
|
||||
if err != nil {
|
||||
return ok, err
|
||||
}
|
||||
if ok {
|
||||
return ok, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (cl CheckerList) Hash() string {
|
||||
var sb strings.Builder
|
||||
|
||||
for _, c := range cl {
|
||||
fmt.Fprintln(&sb, c.Hash())
|
||||
}
|
||||
|
||||
return internal.SHA256sum(sb.String())
|
||||
}
|
||||
|
||||
type staticHashChecker struct {
|
||||
hash string
|
||||
}
|
||||
|
|
@ -57,7 +27,7 @@ func (staticHashChecker) Check(r *http.Request) (bool, error) {
|
|||
|
||||
func (s staticHashChecker) Hash() string { return s.hash }
|
||||
|
||||
func NewStaticHashChecker(hashable string) Checker {
|
||||
func NewStaticHashChecker(hashable string) checker.Impl {
|
||||
return staticHashChecker{hash: internal.SHA256sum(hashable)}
|
||||
}
|
||||
|
||||
|
|
@ -66,7 +36,7 @@ type RemoteAddrChecker struct {
|
|||
hash string
|
||||
}
|
||||
|
||||
func NewRemoteAddrChecker(cidrs []string) (Checker, error) {
|
||||
func NewRemoteAddrChecker(cidrs []string) (checker.Impl, error) {
|
||||
ranger := cidranger.NewPCTrieRanger()
|
||||
var sb strings.Builder
|
||||
|
||||
|
|
@ -122,11 +92,11 @@ type HeaderMatchesChecker struct {
|
|||
hash string
|
||||
}
|
||||
|
||||
func NewUserAgentChecker(rexStr string) (Checker, error) {
|
||||
func NewUserAgentChecker(rexStr string) (checker.Impl, error) {
|
||||
return NewHeaderMatchesChecker("User-Agent", rexStr)
|
||||
}
|
||||
|
||||
func NewHeaderMatchesChecker(header, rexStr string) (Checker, error) {
|
||||
func NewHeaderMatchesChecker(header, rexStr string) (checker.Impl, error) {
|
||||
rex, err := regexp.Compile(strings.TrimSpace(rexStr))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: regex %s failed parse: %w", ErrMisconfiguration, rexStr, err)
|
||||
|
|
@ -151,7 +121,7 @@ type PathChecker struct {
|
|||
hash string
|
||||
}
|
||||
|
||||
func NewPathChecker(rexStr string) (Checker, error) {
|
||||
func NewPathChecker(rexStr string) (checker.Impl, error) {
|
||||
rex, err := regexp.Compile(strings.TrimSpace(rexStr))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: regex %s failed parse: %w", ErrMisconfiguration, rexStr, err)
|
||||
|
|
@ -171,7 +141,7 @@ func (pc *PathChecker) Hash() string {
|
|||
return pc.hash
|
||||
}
|
||||
|
||||
func NewHeaderExistsChecker(key string) Checker {
|
||||
func NewHeaderExistsChecker(key string) checker.Impl {
|
||||
return headerExistsChecker{strings.TrimSpace(key)}
|
||||
}
|
||||
|
||||
|
|
@ -191,8 +161,8 @@ func (hec headerExistsChecker) Hash() string {
|
|||
return internal.SHA256sum(hec.header)
|
||||
}
|
||||
|
||||
func NewHeadersChecker(headermap map[string]string) (Checker, error) {
|
||||
var result CheckerList
|
||||
func NewHeadersChecker(headermap map[string]string) (checker.Impl, error) {
|
||||
var result checker.List
|
||||
var errs []error
|
||||
|
||||
for key, rexStr := range headermap {
|
||||
|
|
|
|||
41
lib/policy/checker/checker.go
Normal file
41
lib/policy/checker/checker.go
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
// Package checker defines the Checker interface and a helper utility to avoid import cycles.
|
||||
package checker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/TecharoHQ/anubis/internal"
|
||||
)
|
||||
|
||||
type Impl interface {
|
||||
Check(*http.Request) (bool, error)
|
||||
Hash() string
|
||||
}
|
||||
|
||||
type List []Impl
|
||||
|
||||
func (l List) Check(r *http.Request) (bool, error) {
|
||||
for _, c := range l {
|
||||
ok, err := c.Check(r)
|
||||
if err != nil {
|
||||
return ok, err
|
||||
}
|
||||
if ok {
|
||||
return ok, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (l List) Hash() string {
|
||||
var sb strings.Builder
|
||||
|
||||
for _, c := range l {
|
||||
fmt.Fprintln(&sb, c.Hash())
|
||||
}
|
||||
|
||||
return internal.SHA256sum(sb.String())
|
||||
}
|
||||
44
lib/policy/config/asn.go
Normal file
44
lib/policy/config/asn.go
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrPrivateASN = errors.New("bot.ASNs: you have specified a private use ASN")
|
||||
)
|
||||
|
||||
type ASNs struct {
|
||||
Match []uint32 `json:"match"`
|
||||
}
|
||||
|
||||
func (a *ASNs) Valid() error {
|
||||
var errs []error
|
||||
|
||||
for _, asn := range a.Match {
|
||||
if isPrivateASN(asn) {
|
||||
errs = append(errs, fmt.Errorf("%w: %d is private (see RFC 6996)", ErrPrivateASN, asn))
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs) != 0 {
|
||||
return fmt.Errorf("bot.ASNs: invalid ASN settings: %w", errors.Join(errs...))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isPrivateASN checks if an ASN is in the private use area.
|
||||
//
|
||||
// Based on RFC 6996 and IANA allocations.
|
||||
func isPrivateASN(asn uint32) bool {
|
||||
switch {
|
||||
case asn >= 64512 && asn <= 65534:
|
||||
return true
|
||||
case asn >= 4200000000 && asn <= 4294967294:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -55,6 +55,10 @@ type BotConfig struct {
|
|||
Name string `json:"name" yaml:"name"`
|
||||
Action Rule `json:"action" yaml:"action"`
|
||||
RemoteAddr []string `json:"remote_addresses,omitempty" yaml:"remote_addresses,omitempty"`
|
||||
|
||||
// Thoth features
|
||||
GeoIP *GeoIP `json:"geoip,omitempty"`
|
||||
ASNs *ASNs `json:"asns,omitempty"`
|
||||
}
|
||||
|
||||
func (b BotConfig) Zero() bool {
|
||||
|
|
@ -66,6 +70,8 @@ func (b BotConfig) Zero() bool {
|
|||
b.Action != "",
|
||||
len(b.RemoteAddr) != 0,
|
||||
b.Challenge != nil,
|
||||
b.GeoIP != nil,
|
||||
b.ASNs != nil,
|
||||
} {
|
||||
if cond {
|
||||
return false
|
||||
|
|
@ -85,7 +91,9 @@ func (b *BotConfig) Valid() error {
|
|||
allFieldsEmpty := b.UserAgentRegex == nil &&
|
||||
b.PathRegex == nil &&
|
||||
len(b.RemoteAddr) == 0 &&
|
||||
len(b.HeadersRegex) == 0
|
||||
len(b.HeadersRegex) == 0 &&
|
||||
b.ASNs == nil &&
|
||||
b.GeoIP == nil
|
||||
|
||||
if allFieldsEmpty && b.Expression == nil {
|
||||
errs = append(errs, ErrBotMustHaveUserAgentOrPath)
|
||||
|
|
|
|||
36
lib/policy/config/geoip.go
Normal file
36
lib/policy/config/geoip.go
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
countryCodeRegexp = regexp.MustCompile(`^\w{2}$`)
|
||||
|
||||
ErrNotCountryCode = errors.New("config.Bot: invalid country code")
|
||||
)
|
||||
|
||||
type GeoIP struct {
|
||||
Countries []string `json:"countries"`
|
||||
}
|
||||
|
||||
func (g *GeoIP) Valid() error {
|
||||
var errs []error
|
||||
|
||||
for i, cc := range g.Countries {
|
||||
if !countryCodeRegexp.MatchString(cc) {
|
||||
errs = append(errs, fmt.Errorf("%w: %s", ErrNotCountryCode, cc))
|
||||
}
|
||||
|
||||
g.Countries[i] = strings.ToLower(cc)
|
||||
}
|
||||
|
||||
if len(errs) != 0 {
|
||||
return fmt.Errorf("bot.GeoIP: invalid GeoIP settings: %w", errors.Join(errs...))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
6
lib/policy/config/testdata/good/challenge_cloudflare.yaml
vendored
Normal file
6
lib/policy/config/testdata/good/challenge_cloudflare.yaml
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
bots:
|
||||
- name: challenge-cloudflare
|
||||
action: CHALLENGE
|
||||
asns:
|
||||
match:
|
||||
- 13335 # Cloudflare
|
||||
6
lib/policy/config/testdata/good/geoip_us.yaml
vendored
Normal file
6
lib/policy/config/testdata/good/geoip_us.yaml
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
bots:
|
||||
- name: compute-tarrif-us
|
||||
action: CHALLENGE
|
||||
geoip:
|
||||
countries:
|
||||
- US
|
||||
|
|
@ -1,10 +1,14 @@
|
|||
package policy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
|
||||
"github.com/TecharoHQ/anubis/internal/thoth"
|
||||
"github.com/TecharoHQ/anubis/lib/policy/checker"
|
||||
"github.com/TecharoHQ/anubis/lib/policy/config"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
|
|
@ -35,7 +39,7 @@ func NewParsedConfig(orig *config.Config) *ParsedConfig {
|
|||
}
|
||||
}
|
||||
|
||||
func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) {
|
||||
func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) {
|
||||
c, err := config.Load(fin, fname)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -43,6 +47,8 @@ func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedCon
|
|||
|
||||
var validationErrs []error
|
||||
|
||||
tc, hasThothClient := thoth.FromContext(ctx)
|
||||
|
||||
result := NewParsedConfig(c)
|
||||
result.DefaultDifficulty = defaultDifficulty
|
||||
|
||||
|
|
@ -57,7 +63,7 @@ func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedCon
|
|||
Action: b.Action,
|
||||
}
|
||||
|
||||
cl := CheckerList{}
|
||||
cl := checker.List{}
|
||||
|
||||
if len(b.RemoteAddr) > 0 {
|
||||
c, err := NewRemoteAddrChecker(b.RemoteAddr)
|
||||
|
|
@ -104,6 +110,24 @@ func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedCon
|
|||
}
|
||||
}
|
||||
|
||||
if b.ASNs != nil {
|
||||
if !hasThothClient {
|
||||
slog.Warn("You have specified a Thoth specific check but you have no Thoth client configured. Please read https://anubis.techaro.lol/docs/admin/thoth for more information", "check", "asn", "settings", b.ASNs)
|
||||
continue
|
||||
}
|
||||
|
||||
cl = append(cl, tc.ASNCheckerFor(b.ASNs.Match))
|
||||
}
|
||||
|
||||
if b.GeoIP != nil {
|
||||
if !hasThothClient {
|
||||
slog.Warn("You have specified a Thoth specific check but you have no Thoth client configured. Please read https://anubis.techaro.lol/docs/admin/thoth for more information", "check", "geoip", "settings", b.GeoIP)
|
||||
continue
|
||||
}
|
||||
|
||||
cl = append(cl, tc.GeoIPCheckerFor(b.GeoIP.Countries))
|
||||
}
|
||||
|
||||
if b.Challenge == nil {
|
||||
parsedBot.Challenge = &config.ChallengeRules{
|
||||
Difficulty: defaultDifficulty,
|
||||
|
|
|
|||
|
|
@ -7,21 +7,25 @@ import (
|
|||
|
||||
"github.com/TecharoHQ/anubis"
|
||||
"github.com/TecharoHQ/anubis/data"
|
||||
"github.com/TecharoHQ/anubis/internal/thoth/thothmock"
|
||||
)
|
||||
|
||||
func TestDefaultPolicyMustParse(t *testing.T) {
|
||||
ctx := thothmock.WithMockThoth(t)
|
||||
|
||||
fin, err := data.BotPolicies.Open("botPolicies.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer fin.Close()
|
||||
|
||||
if _, err := ParseConfig(fin, "botPolicies.json", anubis.DefaultDifficulty); err != nil {
|
||||
if _, err := ParseConfig(ctx, fin, "botPolicies.json", anubis.DefaultDifficulty); err != nil {
|
||||
t.Fatalf("can't parse config: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGoodConfigs(t *testing.T) {
|
||||
|
||||
finfos, err := os.ReadDir("config/testdata/good")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
@ -30,20 +34,37 @@ func TestGoodConfigs(t *testing.T) {
|
|||
for _, st := range finfos {
|
||||
st := st
|
||||
t.Run(st.Name(), func(t *testing.T) {
|
||||
fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer fin.Close()
|
||||
t.Run("with-thoth", func(t *testing.T) {
|
||||
fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer fin.Close()
|
||||
|
||||
if _, err := ParseConfig(fin, fin.Name(), anubis.DefaultDifficulty); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := thothmock.WithMockThoth(t)
|
||||
if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("without-thoth", func(t *testing.T) {
|
||||
fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer fin.Close()
|
||||
|
||||
if _, err := ParseConfig(t.Context(), fin, fin.Name(), anubis.DefaultDifficulty); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBadConfigs(t *testing.T) {
|
||||
ctx := thothmock.WithMockThoth(t)
|
||||
|
||||
finfos, err := os.ReadDir("config/testdata/bad")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
@ -58,7 +79,7 @@ func TestBadConfigs(t *testing.T) {
|
|||
}
|
||||
defer fin.Close()
|
||||
|
||||
if _, err := ParseConfig(fin, fin.Name(), anubis.DefaultDifficulty); err == nil {
|
||||
if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty); err == nil {
|
||||
t.Fatal(err)
|
||||
} else {
|
||||
t.Log(err)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue