feat(lib): use new challenge creation flow (#749)

* feat(decaymap): add Delete method

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(lib/challenge): refactor Validate to take ValidateInput

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(lib): implement store interface

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(lib/store): all metapackage to import all store implementations

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(policy): import all store backends

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(lib): use new challenge creation flow

Previously Anubis constructed challenge strings from request metadata.
This was a good idea in spirit, but has turned out to be a very bad idea
in practice. This new flow reuses the Store facility to dynamically
create challenge values with completely random data.

This is a fairly big rewrite of how Anubis processes challenges. Right
now it defaults to using the in-memory storage backend, but on-disk
(boltdb) and valkey-based adaptors will come soon.

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(decaymap): fix documentation typo

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(lib): fix SA4004

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(lib/store): make generic storage interface test adaptor

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(decaymap): invert locking process for Delete

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(lib/store): add bbolt store implementation

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: go mod tidy

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(devcontainer): adapt to docker compose, add valkey service

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(lib): make challenges live for 30 minutes by default

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(lib/store): implement valkey backend

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(lib/store/valkey): disable tests if not using docker

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(lib/policy/config): ensure valkey stores can be loaded

Signed-off-by: Xe Iaso <me@xeiaso.net>

* Update metadata

check-spelling run (pull_request) for Xe/store-interface

Signed-off-by: check-spelling-bot <check-spelling-bot@users.noreply.github.com>
on-behalf-of: @check-spelling <check-spelling-bot@check-spelling.dev>

* chore(devcontainer): remove port forwards because vs code handles that for you

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs(default-config): add a nudge to the storage backends section of the docs

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(docs): listen on 0.0.0.0 for dev container support

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs(policy): document storage backends

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs: update CHANGELOG and internal links

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs(admin/policies): don't start a sentence with as

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: fixes found in review

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
Signed-off-by: check-spelling-bot <check-spelling-bot@users.noreply.github.com>
This commit is contained in:
Xe Iaso 2025-07-04 20:42:28 +00:00 committed by GitHub
parent 506d8817d5
commit dff2176beb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
43 changed files with 1539 additions and 140 deletions

View file

@ -1,8 +1,9 @@
package lib
import (
"context"
"crypto/ed25519"
"crypto/sha256"
"crypto/rand"
"encoding/json"
"errors"
"fmt"
@ -16,6 +17,7 @@ import (
"github.com/golang-jwt/jwt/v5"
"github.com/google/cel-go/common/types"
"github.com/google/uuid"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -30,6 +32,7 @@ import (
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/store"
// challenge implementations
_ "github.com/TecharoHQ/anubis/lib/challenge/metarefresh"
@ -72,6 +75,7 @@ type Server struct {
ed25519Priv ed25519.PrivateKey
hs512Secret []byte
opts Options
store store.Interface
}
func (s *Server) getTokenKeyfunc() jwt.Keyfunc {
@ -87,23 +91,51 @@ func (s *Server) getTokenKeyfunc() jwt.Keyfunc {
}
}
func (s *Server) challengeFor(r *http.Request, difficulty int) string {
var fp [32]byte
if len(s.hs512Secret) == 0 {
fp = sha256.Sum256(s.ed25519Priv.Public().(ed25519.PublicKey)[:])
} else {
fp = sha256.Sum256(s.hs512Secret)
func (s *Server) challengeFor(r *http.Request) (*challenge.Challenge, error) {
ckies := r.CookiesNamed(anubis.TestCookieName)
if len(ckies) == 0 {
return s.issueChallenge(r.Context(), r)
}
challengeData := fmt.Sprintf(
"X-Real-IP=%s,User-Agent=%s,WeekTime=%s,Fingerprint=%x,Difficulty=%d",
r.Header.Get("X-Real-Ip"),
r.UserAgent(),
time.Now().UTC().Round(24*7*time.Hour).Format(time.RFC3339),
fp,
difficulty,
)
return internal.FastHash(challengeData)
j := store.JSON[challenge.Challenge]{Underlying: s.store}
ckie := ckies[0]
chall, err := j.Get(r.Context(), "challenge:"+ckie.Value)
if err != nil {
return nil, err
}
return &chall, nil
}
func (s *Server) issueChallenge(ctx context.Context, r *http.Request) (*challenge.Challenge, error) {
id, err := uuid.NewV7()
if err != nil {
return nil, err
}
var randomData = make([]byte, 256)
if _, err := rand.Read(randomData); err != nil {
return nil, err
}
chall := challenge.Challenge{
ID: id.String(),
RandomData: fmt.Sprintf("%x", randomData),
IssuedAt: time.Now(),
Metadata: map[string]string{
"User-Agent": r.Header.Get("User-Agent"),
"X-Real-Ip": r.Header.Get("X-Real-Ip"),
},
}
j := store.JSON[challenge.Challenge]{Underlying: s.store}
if err := j.Set(ctx, "challenge:"+id.String(), chall, 30*time.Minute); err != nil {
return nil, err
}
return &chall, err
}
func (s *Server) maybeReverseProxyHttpStatusOnly(w http.ResponseWriter, r *http.Request) {
@ -309,15 +341,30 @@ func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
return
}
lg = lg.With("check_result", cr)
chal := s.challengeFor(r, rule.Challenge.Difficulty)
s.SetCookie(w, CookieOpts{Host: r.Host, Name: anubis.TestCookieName, Value: chal})
chall, err := s.challengeFor(r)
if err != nil {
lg.Error("failed to fetch or issue challenge", "err", err)
w.WriteHeader(http.StatusInternalServerError)
err := encoder.Encode(struct {
Error string `json:"error"`
}{
Error: fmt.Sprintf("%s \"makeChallenge\"", localizer.T("internal_server_error")),
})
if err != nil {
lg.Error("failed to encode error response", "err", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
s.SetCookie(w, CookieOpts{Host: r.Host, Name: anubis.TestCookieName, Value: chall.ID})
err = encoder.Encode(struct {
Rules *config.ChallengeRules `json:"rules"`
Challenge string `json:"challenge"`
}{
Challenge: chal,
Challenge: chall.RandomData,
Rules: rule.Challenge,
})
if err != nil {
@ -325,7 +372,7 @@ func (s *Server) MakeChallenge(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
return
}
lg.Debug("made challenge", "challenge", chal, "rules", rule.Challenge, "cr", cr)
lg.Debug("made challenge", "challenge", chall, "rules", rule.Challenge, "cr", cr)
challengesIssued.WithLabelValues("api").Inc()
}
@ -384,9 +431,20 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
return
}
challengeStr := s.challengeFor(r, rule.Challenge.Difficulty)
chall, err := s.challengeFor(r)
if err != nil {
lg.Error("check failed", "err", err)
s.respondWithError(w, r, fmt.Sprintf("%s: %s", localizer.T("internal_server_error"), rule.Challenge.Algorithm))
return
}
if err := impl.Validate(r, lg, rule, challengeStr); err != nil {
in := &challenge.ValidateInput{
Challenge: chall,
Rule: rule,
Store: s.store,
}
if err := impl.Validate(r, lg, in); err != nil {
failedValidations.WithLabelValues(rule.Challenge.Algorithm).Inc()
var cerr *challenge.Error
s.ClearCookie(w, CookieOpts{Path: cookiePath, Host: r.Host})
@ -405,7 +463,7 @@ func (s *Server) PassChallenge(w http.ResponseWriter, r *http.Request) {
// generate JWT cookie
tokenString, err := s.signJWT(jwt.MapClaims{
"challenge": challengeStr,
"challenge": chall.ID,
"method": rule.Challenge.Algorithm,
"policyRule": rule.Hash(),
"action": string(cr.Rule),

View file

@ -1,60 +1,11 @@
package challenge
import (
"log/slog"
"net/http"
"sort"
"sync"
import "time"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/a-h/templ"
)
var (
registry map[string]Impl = map[string]Impl{}
regLock sync.RWMutex
)
func Register(name string, impl Impl) {
regLock.Lock()
defer regLock.Unlock()
registry[name] = impl
}
func Get(name string) (Impl, bool) {
regLock.RLock()
defer regLock.RUnlock()
result, ok := registry[name]
return result, ok
}
func Methods() []string {
regLock.RLock()
defer regLock.RUnlock()
var result []string
for method := range registry {
result = append(result, method)
}
sort.Strings(result)
return result
}
type IssueInput struct {
Impressum *config.Impressum
Rule *policy.Bot
Challenge string
OGTags map[string]string
}
type Impl interface {
// Setup registers any additional routes with the Impl for assets or API routes.
Setup(mux *http.ServeMux)
// Issue a new challenge to the user, called by the Anubis.
Issue(r *http.Request, lg *slog.Logger, in *IssueInput) (templ.Component, error)
// Validate a challenge, making sure that it passes muster.
Validate(r *http.Request, lg *slog.Logger, rule *policy.Bot, challenge string) error
// Challenge is the metadata about a single challenge issuance.
type Challenge struct {
ID string `json:"id"` // UUID identifying the challenge
RandomData string `json:"randomData"` // The random data the client processes
IssuedAt time.Time `json:"issuedAt"` // When the challenge was issued
Metadata map[string]string `json:"metadata"` // Challenge metadata such as IP address and user agent
}

View file

@ -0,0 +1,23 @@
package challengetest
import (
"testing"
"time"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/challenge"
"github.com/google/uuid"
)
func New(t *testing.T) *challenge.Challenge {
t.Helper()
id := uuid.Must(uuid.NewV7())
randomData := internal.SHA256sum(time.Now().String())
return &challenge.Challenge{
ID: id.String(),
RandomData: randomData,
IssuedAt: time.Now(),
}
}

View file

@ -0,0 +1,7 @@
package challengetest
import "testing"
func TestNew(t *testing.T) {
_ = New(t)
}

View file

@ -0,0 +1,68 @@
package challenge
import (
"log/slog"
"net/http"
"sort"
"sync"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/store"
"github.com/a-h/templ"
)
var (
registry map[string]Impl = map[string]Impl{}
regLock sync.RWMutex
)
func Register(name string, impl Impl) {
regLock.Lock()
defer regLock.Unlock()
registry[name] = impl
}
func Get(name string) (Impl, bool) {
regLock.RLock()
defer regLock.RUnlock()
result, ok := registry[name]
return result, ok
}
func Methods() []string {
regLock.RLock()
defer regLock.RUnlock()
var result []string
for method := range registry {
result = append(result, method)
}
sort.Strings(result)
return result
}
type IssueInput struct {
Impressum *config.Impressum
Rule *policy.Bot
Challenge *Challenge
OGTags map[string]string
Store store.Interface
}
type ValidateInput struct {
Rule *policy.Bot
Challenge *Challenge
Store store.Interface
}
type Impl interface {
// Setup registers any additional routes with the Impl for assets or API routes.
Setup(mux *http.ServeMux)
// Issue a new challenge to the user, called by the Anubis.
Issue(r *http.Request, lg *slog.Logger, in *IssueInput) (templ.Component, error)
// Validate a challenge, making sure that it passes muster.
Validate(r *http.Request, lg *slog.Logger, in *ValidateInput) error
}

View file

@ -9,7 +9,6 @@ import (
"github.com/TecharoHQ/anubis"
"github.com/TecharoHQ/anubis/lib/challenge"
"github.com/TecharoHQ/anubis/lib/localization"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/web"
"github.com/a-h/templ"
)
@ -32,11 +31,11 @@ func (i *Impl) Issue(r *http.Request, lg *slog.Logger, in *challenge.IssueInput)
q := u.Query()
q.Set("redir", r.URL.String())
q.Set("challenge", in.Challenge)
q.Set("challenge", in.Challenge.RandomData)
u.RawQuery = q.Encode()
loc := localization.GetLocalizer(r)
component, err := web.BaseWithChallengeAndOGTags(loc.T("making_sure_not_bot"), page(in.Challenge, u.String(), in.Rule.Challenge.Difficulty, loc), in.Impressum, in.Challenge, in.Rule.Challenge, in.OGTags, loc)
component, err := web.BaseWithChallengeAndOGTags(loc.T("making_sure_not_bot"), page(u.String(), in.Rule.Challenge.Difficulty, loc), in.Impressum, in.Challenge.RandomData, in.Rule.Challenge, in.OGTags, loc)
if err != nil {
return nil, fmt.Errorf("can't render page: %w", err)
@ -45,11 +44,11 @@ func (i *Impl) Issue(r *http.Request, lg *slog.Logger, in *challenge.IssueInput)
return component, nil
}
func (i *Impl) Validate(r *http.Request, lg *slog.Logger, rule *policy.Bot, wantChallenge string) error {
func (i *Impl) Validate(r *http.Request, lg *slog.Logger, in *challenge.ValidateInput) error {
gotChallenge := r.FormValue("challenge")
if subtle.ConstantTimeCompare([]byte(wantChallenge), []byte(gotChallenge)) != 1 {
return challenge.NewError("validate", "invalid response", fmt.Errorf("%w: wanted response %s but got %s", challenge.ErrFailed, wantChallenge, gotChallenge))
if subtle.ConstantTimeCompare([]byte(in.Challenge.RandomData), []byte(gotChallenge)) != 1 {
return challenge.NewError("validate", "invalid response", fmt.Errorf("%w: wanted response %s but got %s", challenge.ErrFailed, in.Challenge.RandomData, gotChallenge))
}
return nil

View file

@ -7,7 +7,7 @@ import (
"github.com/TecharoHQ/anubis/lib/localization"
)
templ page(challenge, redir string, difficulty int, loc *localization.SimpleLocalizer) {
templ page(redir string, difficulty int, loc *localization.SimpleLocalizer) {
<div class="centered-div">
<img id="image" style="width:100%;max-width:256px;" src={ anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" + anubis.Version }/>
<img style="display:none;" style="width:100%;max-width:256px;" src={ anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/happy.webp?cacheBuster=" + anubis.Version }/>

View file

@ -15,7 +15,7 @@ import (
"github.com/TecharoHQ/anubis/lib/localization"
)
func page(challenge, redir string, difficulty int, loc *localization.SimpleLocalizer) templ.Component {
func page(redir string, difficulty int, loc *localization.SimpleLocalizer) templ.Component {
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {

View file

@ -11,7 +11,6 @@ import (
"github.com/TecharoHQ/anubis/internal"
chall "github.com/TecharoHQ/anubis/lib/challenge"
"github.com/TecharoHQ/anubis/lib/localization"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/web"
"github.com/a-h/templ"
)
@ -31,7 +30,7 @@ func (i *Impl) Setup(mux *http.ServeMux) {
func (i *Impl) Issue(r *http.Request, lg *slog.Logger, in *chall.IssueInput) (templ.Component, error) {
loc := localization.GetLocalizer(r)
component, err := web.BaseWithChallengeAndOGTags(loc.T("making_sure_not_bot"), web.Index(loc), in.Impressum, in.Challenge, in.Rule.Challenge, in.OGTags, loc)
component, err := web.BaseWithChallengeAndOGTags(loc.T("making_sure_not_bot"), web.Index(loc), in.Impressum, in.Challenge.RandomData, in.Rule.Challenge, in.OGTags, loc)
if err != nil {
return nil, fmt.Errorf("can't render page: %w", err)
}
@ -39,7 +38,10 @@ func (i *Impl) Issue(r *http.Request, lg *slog.Logger, in *chall.IssueInput) (te
return component, nil
}
func (i *Impl) Validate(r *http.Request, lg *slog.Logger, rule *policy.Bot, challenge string) error {
func (i *Impl) Validate(r *http.Request, lg *slog.Logger, in *chall.ValidateInput) error {
rule := in.Rule
challenge := in.Challenge.RandomData
nonceStr := r.FormValue("nonce")
if nonceStr == "" {
return chall.NewError("validate", "invalid response", fmt.Errorf("%w nonce", chall.ErrMissingField))

View file

@ -124,16 +124,25 @@ func TestBasic(t *testing.T) {
t.Run(cs.name, func(t *testing.T) {
lg := slog.With()
i.Setup(http.NewServeMux())
inp := &challenge.IssueInput{
Rule: bot,
Challenge: cs.challengeStr,
Rule: bot,
Challenge: &challenge.Challenge{
RandomData: cs.challengeStr,
},
}
if _, err := i.Issue(cs.req, lg, inp); err != nil {
t.Errorf("can't issue challenge: %v", err)
}
if err := i.Validate(cs.req, lg, bot, cs.challengeStr); !errors.Is(err, cs.err) {
if err := i.Validate(cs.req, lg, &challenge.ValidateInput{
Rule: bot,
Challenge: &challenge.Challenge{
RandomData: cs.challengeStr,
},
}); !errors.Is(err, cs.err) {
t.Errorf("got wrong error from Validate, got %v but wanted %v", err, cs.err)
}
})

View file

@ -110,6 +110,7 @@ func New(opts Options) (*Server, error) {
opts: opts,
DNSBLCache: decaymap.New[string, dnsbl.DroneBLResponse](),
OGTags: ogtags.NewOGTagCache(opts.Target, opts.Policy.OpenGraph),
store: opts.Policy.Store,
}
mux := http.NewServeMux()

View file

@ -128,7 +128,12 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
}
challengesIssued.WithLabelValues("embedded").Add(1)
challengeStr := s.challengeFor(r, rule.Challenge.Difficulty)
chall, err := s.challengeFor(r)
if err != nil {
lg.Error("can't get challenge", "err", "err")
s.respondWithError(w, r, fmt.Sprintf("%s: %s", localizer.T("internal_server_error"), rule.Challenge.Algorithm))
return
}
var ogTags map[string]string = nil
if s.opts.OpenGraph.Enabled {
@ -140,7 +145,7 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
}
s.SetCookie(w, CookieOpts{
Value: challengeStr,
Value: chall.ID,
Host: r.Host,
Path: "/",
Name: anubis.TestCookieName,
@ -157,8 +162,9 @@ func (s *Server) RenderIndex(w http.ResponseWriter, r *http.Request, rule *polic
in := &challenge.IssueInput{
Impressum: s.policy.Impressum,
Rule: rule,
Challenge: challengeStr,
Challenge: chall,
OGTags: ogTags,
Store: s.store,
}
component, err := impl.Issue(r, lg, in)

View file

@ -329,6 +329,7 @@ type fileConfig struct {
OpenGraph openGraphFileConfig `json:"openGraph,omitempty"`
Impressum *Impressum `json:"impressum,omitempty"`
StatusCodes StatusCodes `json:"status_codes"`
Store *Store `json:"store"`
Thresholds []Threshold `json:"thresholds"`
}
@ -361,6 +362,12 @@ func (c *fileConfig) Valid() error {
}
}
if c.Store != nil {
if err := c.Store.Valid(); err != nil {
errs = append(errs, err)
}
}
if len(errs) != 0 {
return fmt.Errorf("config is not valid:\n%w", errors.Join(errs...))
}
@ -374,6 +381,9 @@ func Load(fin io.Reader, fname string) (*Config, error) {
Challenge: http.StatusOK,
Deny: http.StatusOK,
},
Store: &Store{
Backend: "memory",
},
}
if err := yaml.NewYAMLToJSONDecoder(fin).Decode(&c); err != nil {
@ -392,6 +402,7 @@ func Load(fin io.Reader, fname string) (*Config, error) {
Override: c.OpenGraph.Override,
},
StatusCodes: c.StatusCodes,
Store: c.Store,
}
if c.OpenGraph.TimeToLive != "" {
@ -457,6 +468,7 @@ type Config struct {
Impressum *Impressum
OpenGraph OpenGraph
StatusCodes StatusCodes
Store *Store
}
func (c Config) Valid() error {

View file

@ -1,4 +1,4 @@
package config
package config_test
import (
"errors"
@ -8,6 +8,7 @@ import (
"testing"
"github.com/TecharoHQ/anubis/data"
. "github.com/TecharoHQ/anubis/lib/policy/config"
)
func p[V any](v V) *V { return &v }
@ -325,37 +326,37 @@ func TestConfigValidBad(t *testing.T) {
func TestBotConfigZero(t *testing.T) {
var b BotConfig
if !b.Zero() {
t.Error("zero value BotConfig is not zero value")
t.Error("zero value config.BotConfig is not zero value")
}
b.Name = "hi"
if b.Zero() {
t.Error("BotConfig with name is zero value")
t.Error("config.BotConfig with name is zero value")
}
b.UserAgentRegex = p(".*")
if b.Zero() {
t.Error("BotConfig with user agent regex is zero value")
t.Error("config.BotConfig with user agent regex is zero value")
}
b.PathRegex = p(".*")
if b.Zero() {
t.Error("BotConfig with path regex is zero value")
t.Error("config.BotConfig with path regex is zero value")
}
b.HeadersRegex = map[string]string{"hi": "there"}
if b.Zero() {
t.Error("BotConfig with headers regex is zero value")
t.Error("config.BotConfig with headers regex is zero value")
}
b.Action = RuleAllow
if b.Zero() {
t.Error("BotConfig with action is zero value")
t.Error("config.BotConfig with action is zero value")
}
b.RemoteAddr = []string{"::/0"}
if b.Zero() {
t.Error("BotConfig with remote addresses is zero value")
t.Error("config.BotConfig with remote addresses is zero value")
}
b.Challenge = &ChallengeRules{
@ -364,6 +365,6 @@ func TestBotConfigZero(t *testing.T) {
Algorithm: DefaultAlgorithm,
}
if b.Zero() {
t.Error("BotConfig with challenge rules is zero value")
t.Error("config.BotConfig with challenge rules is zero value")
}
}

View file

@ -0,0 +1,44 @@
package config
import (
"encoding/json"
"errors"
"fmt"
"github.com/TecharoHQ/anubis/lib/store"
_ "github.com/TecharoHQ/anubis/lib/store/all"
)
var (
ErrNoStoreBackend = errors.New("config.Store: no backend defined")
ErrUnknownStoreBackend = errors.New("config.Store: unknown backend")
)
type Store struct {
Backend string `json:"backend"`
Parameters json.RawMessage `json:"parameters"`
}
func (s *Store) Valid() error {
var errs []error
if len(s.Backend) == 0 {
errs = append(errs, ErrNoStoreBackend)
}
fac, ok := store.Get(s.Backend)
switch ok {
case true:
if err := fac.Valid(s.Parameters); err != nil {
errs = append(errs, err)
}
case false:
errs = append(errs, fmt.Errorf("%w: %q", ErrUnknownStoreBackend, s.Backend))
}
if len(errs) != 0 {
return errors.Join(errs...)
}
return nil
}

View file

@ -0,0 +1,84 @@
package config_test
import (
"encoding/json"
"errors"
"testing"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/store/bbolt"
"github.com/TecharoHQ/anubis/lib/store/valkey"
)
func TestStoreValid(t *testing.T) {
for _, tt := range []struct {
name string
input config.Store
err error
}{
{
name: "no backend",
input: config.Store{},
err: config.ErrNoStoreBackend,
},
{
name: "in-memory backend",
input: config.Store{
Backend: "memory",
},
},
{
name: "bbolt backend",
input: config.Store{
Backend: "bbolt",
Parameters: json.RawMessage(`{"path": "/tmp/foo", "bucket": "bar"}`),
},
},
{
name: "valkey backend",
input: config.Store{
Backend: "valkey",
Parameters: json.RawMessage(`{"url": "redis://valkey:6379/0"}`),
},
},
{
name: "valkey backend no URL",
input: config.Store{
Backend: "valkey",
Parameters: json.RawMessage(`{}`),
},
err: valkey.ErrNoURL,
},
{
name: "valkey backend bad URL",
input: config.Store{
Backend: "valkey",
Parameters: json.RawMessage(`{"url": "http://anubis.techaro.lol"}`),
},
err: valkey.ErrBadURL,
},
{
name: "bbolt backend no path",
input: config.Store{
Backend: "bbolt",
Parameters: json.RawMessage(`{"path": "", "bucket": "bar"}`),
},
err: bbolt.ErrMissingPath,
},
{
name: "unknown backend",
input: config.Store{
Backend: "taco salad",
},
err: config.ErrUnknownStoreBackend,
},
} {
t.Run(tt.name, func(t *testing.T) {
if err := tt.input.Valid(); !errors.Is(err, tt.err) {
t.Logf("want: %v", tt.err)
t.Logf("got: %v", err)
t.Error("invalid error returned")
}
})
}
}

View file

@ -11,8 +11,11 @@ import (
"github.com/TecharoHQ/anubis/internal/thoth"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/store"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
_ "github.com/TecharoHQ/anubis/lib/store/all"
)
var (
@ -35,9 +38,10 @@ type ParsedConfig struct {
OpenGraph config.OpenGraph
DefaultDifficulty int
StatusCodes config.StatusCodes
Store store.Interface
}
func NewParsedConfig(orig *config.Config) *ParsedConfig {
func newParsedConfig(orig *config.Config) *ParsedConfig {
return &ParsedConfig{
orig: orig,
OpenGraph: orig.OpenGraph,
@ -55,7 +59,7 @@ func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDiffic
tc, hasThothClient := thoth.FromContext(ctx)
result := NewParsedConfig(c)
result := newParsedConfig(c)
result.DefaultDifficulty = defaultDifficulty
for _, b := range c.Bots {
@ -178,6 +182,19 @@ func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDiffic
result.Thresholds = append(result.Thresholds, threshold)
}
stFac, ok := store.Get(c.Store.Backend)
switch ok {
case true:
store, err := stFac.Build(ctx, c.Store.Parameters)
if err != nil {
validationErrs = append(validationErrs, err)
} else {
result.Store = store
}
case false:
validationErrs = append(validationErrs, config.ErrUnknownStoreBackend)
}
if len(validationErrs) > 0 {
return nil, fmt.Errorf("errors validating policy config JSON %s: %w", fname, errors.Join(validationErrs...))
}

10
lib/store/all/all.go Normal file
View file

@ -0,0 +1,10 @@
// Package all is a meta-package that imports all store implementations.
//
// This is a HACK to make tests work consistently.
package all
import (
_ "github.com/TecharoHQ/anubis/lib/store/bbolt"
_ "github.com/TecharoHQ/anubis/lib/store/memory"
_ "github.com/TecharoHQ/anubis/lib/store/valkey"
)

142
lib/store/bbolt/bbolt.go Normal file
View file

@ -0,0 +1,142 @@
package bbolt
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"time"
"github.com/TecharoHQ/anubis/lib/store"
"go.etcd.io/bbolt"
)
var (
ErrBucketDoesNotExist = errors.New("bbolt: bucket does not exist")
ErrNotExists = errors.New("bbolt: value does not exist in store")
)
type Item struct {
Data []byte `json:"data"`
Expires time.Time `json:"expires"`
}
type Store struct {
bucket []byte
bdb *bbolt.DB
}
func (s *Store) Delete(ctx context.Context, key string) error {
return s.bdb.Update(func(tx *bbolt.Tx) error {
bkt := tx.Bucket(s.bucket)
if bkt == nil {
return fmt.Errorf("%w: %q", ErrBucketDoesNotExist, string(s.bucket))
}
if bkt.Get([]byte(key)) == nil {
return fmt.Errorf("%w: %q", ErrNotExists, key)
}
return bkt.Delete([]byte(key))
})
}
func (s *Store) Get(ctx context.Context, key string) ([]byte, error) {
var i Item
if err := s.bdb.View(func(tx *bbolt.Tx) error {
bkt := tx.Bucket(s.bucket)
if bkt == nil {
return fmt.Errorf("%w: %q", ErrBucketDoesNotExist, string(s.bucket))
}
bucketData := bkt.Get([]byte(key))
if bucketData == nil {
return fmt.Errorf("%w: %q", store.ErrNotFound, key)
}
if err := json.Unmarshal(bucketData, &i); err != nil {
return fmt.Errorf("%w: %w", store.ErrCantDecode, err)
}
return nil
}); err != nil {
return nil, err
}
if time.Now().After(i.Expires) {
go s.Delete(context.Background(), key)
return nil, fmt.Errorf("%w: %q", store.ErrNotFound, key)
}
return i.Data, nil
}
func (s *Store) Set(ctx context.Context, key string, value []byte, expiry time.Duration) error {
i := Item{
Data: value,
Expires: time.Now().Add(expiry),
}
data, err := json.Marshal(i)
if err != nil {
return fmt.Errorf("%w: %w", store.ErrCantEncode, err)
}
return s.bdb.Update(func(tx *bbolt.Tx) error {
bkt := tx.Bucket(s.bucket)
if bkt == nil {
return fmt.Errorf("%w: %q", ErrBucketDoesNotExist, string(s.bucket))
}
return bkt.Put([]byte(key), data)
})
}
func (s *Store) cleanup(ctx context.Context) error {
now := time.Now()
return s.bdb.Update(func(tx *bbolt.Tx) error {
bkt := tx.Bucket(s.bucket)
if bkt == nil {
return fmt.Errorf("cache bucket %q does not exist", string(s.bucket))
}
return bkt.ForEach(func(k, v []byte) error {
var i Item
data := bkt.Get(k)
if data == nil {
return fmt.Errorf("%s in Cache bucket does not exist???", string(k))
}
if err := json.Unmarshal(data, &i); err != nil {
return fmt.Errorf("can't unmarshal data at key %s: %w", string(k), err)
}
if now.After(i.Expires) {
return bkt.Delete(k)
}
return nil
})
})
}
func (s *Store) cleanupThread(ctx context.Context) {
t := time.NewTicker(5 * time.Minute)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if err := s.cleanup(ctx); err != nil {
slog.Error("error during bbolt cleanup", "err", err)
}
}
}
}

View file

@ -0,0 +1,23 @@
package bbolt
import (
"encoding/json"
"path/filepath"
"testing"
"github.com/TecharoHQ/anubis/lib/store/storetest"
)
func TestImpl(t *testing.T) {
path := filepath.Join(t.TempDir(), "db")
t.Log(path)
data, err := json.Marshal(Config{
Path: path,
Bucket: "anubis",
})
if err != nil {
t.Fatal(err)
}
storetest.Common(t, Factory{}, json.RawMessage(data))
}

100
lib/store/bbolt/factory.go Normal file
View file

@ -0,0 +1,100 @@
package bbolt
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"github.com/TecharoHQ/anubis/lib/store"
"go.etcd.io/bbolt"
)
var (
ErrMissingPath = errors.New("bbolt: path is missing from config")
ErrCantWriteToPath = errors.New("bbolt: can't write to path")
)
func init() {
store.Register("bbolt", Factory{})
}
type Factory struct{}
func (Factory) Build(ctx context.Context, data json.RawMessage) (store.Interface, error) {
var config Config
if err := json.Unmarshal([]byte(data), &config); err != nil {
return nil, fmt.Errorf("%w: %w", store.ErrBadConfig, err)
}
if err := config.Valid(); err != nil {
return nil, fmt.Errorf("%w: %w", store.ErrBadConfig, err)
}
if config.Bucket == "" {
config.Bucket = "anubis"
}
bdb, err := bbolt.Open(config.Path, 0600, nil)
if err != nil {
return nil, fmt.Errorf("can't open bbolt database %s: %w", config.Path, err)
}
if err := bdb.Update(func(tx *bbolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists([]byte(config.Bucket)); err != nil {
return err
}
return nil
}); err != nil {
return nil, fmt.Errorf("can't create bbolt bucket %q: %w", config.Bucket, err)
}
result := &Store{
bdb: bdb,
bucket: []byte(config.Bucket),
}
go result.cleanupThread(ctx)
return result, nil
}
func (Factory) Valid(data json.RawMessage) error {
var config Config
if err := json.Unmarshal([]byte(data), &config); err != nil {
return fmt.Errorf("%w: %w", store.ErrBadConfig, err)
}
if err := config.Valid(); err != nil {
return fmt.Errorf("%w: %w", store.ErrBadConfig, err)
}
return nil
}
type Config struct {
Path string `json:"path"`
Bucket string `json:"bucket,omitempty"`
}
func (c Config) Valid() error {
var errs []error
if c.Path == "" {
errs = append(errs, ErrMissingPath)
} else {
dir := filepath.Dir(c.Path)
if err := os.WriteFile(filepath.Join(dir, ".test-file"), []byte(""), 0600); err != nil {
errs = append(errs, ErrCantWriteToPath)
}
}
if len(errs) != 0 {
return errors.Join(errs...)
}
return nil
}

View file

@ -0,0 +1,50 @@
package bbolt
import (
"encoding/json"
"errors"
"path/filepath"
"testing"
)
func TestFactoryValid(t *testing.T) {
f := Factory{}
t.Run("bad config", func(t *testing.T) {
if err := f.Valid(json.RawMessage(`}`)); err == nil {
t.Error("wanted parsing failure but got a successful result")
}
})
t.Run("invalid config", func(t *testing.T) {
for _, tt := range []struct {
name string
cfg Config
err error
}{
{
name: "missing path",
cfg: Config{},
err: ErrMissingPath,
},
{
name: "unwritable folder",
cfg: Config{
Path: filepath.Join("/", "testdb"),
},
err: ErrCantWriteToPath,
},
} {
t.Run(tt.name, func(t *testing.T) {
data, err := json.Marshal(tt.cfg)
if err != nil {
t.Fatal(err)
}
if err := f.Valid(json.RawMessage(data)); !errors.Is(err, tt.err) {
t.Error(err)
}
})
}
})
}

77
lib/store/interface.go Normal file
View file

@ -0,0 +1,77 @@
package store
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
)
var (
// ErrNotFound is returned when the store implementation cannot find the value
// for a given key.
ErrNotFound = errors.New("store: key not found")
// ErrCantDecode is returned when a store adaptor cannot decode the store format
// to a value used by the code.
ErrCantDecode = errors.New("store: can't decode value")
// ErrCantEncode is returned when a store adaptor cannot encode the value into
// the format that the store uses.
ErrCantEncode = errors.New("store: can't encode value")
// ErrBadConfig is returned when a store adaptor's configuration is invalid.
ErrBadConfig = errors.New("store: configuration is invalid")
)
// Interface defines the calls that Anubis uses for storage in a local or remote
// datastore. This can be implemented with an in-memory, on-disk, or in-database
// storage backend.
type Interface interface {
// Delete removes a value from the store by key.
Delete(ctx context.Context, key string) error
// Get returns the value of a key assuming that value exists and has not expired.
Get(ctx context.Context, key string) ([]byte, error)
// Set puts a value into the store that expires according to its expiry.
Set(ctx context.Context, key string, value []byte, expiry time.Duration) error
}
func z[T any]() T { return *new(T) }
type JSON[T any] struct {
Underlying Interface
}
func (j *JSON[T]) Delete(ctx context.Context, key string) error {
return j.Underlying.Delete(ctx, key)
}
func (j *JSON[T]) Get(ctx context.Context, key string) (T, error) {
data, err := j.Underlying.Get(ctx, key)
if err != nil {
return z[T](), err
}
var result T
if err := json.Unmarshal(data, &result); err != nil {
return z[T](), fmt.Errorf("%w: %w", ErrCantDecode, err)
}
return result, nil
}
func (j *JSON[T]) Set(ctx context.Context, key string, value T, expiry time.Duration) error {
data, err := json.Marshal(value)
if err != nil {
return fmt.Errorf("%w: %w", ErrCantEncode, err)
}
if err := j.Underlying.Set(ctx, key, data, expiry); err != nil {
return err
}
return nil
}

View file

@ -0,0 +1,74 @@
package memory
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/TecharoHQ/anubis/decaymap"
"github.com/TecharoHQ/anubis/lib/store"
)
type factory struct{}
func (factory) Build(ctx context.Context, _ json.RawMessage) (store.Interface, error) {
return New(ctx), nil
}
func (factory) Valid(json.RawMessage) error { return nil }
func init() {
store.Register("memory", factory{})
}
type impl struct {
store *decaymap.Impl[string, []byte]
}
func (i *impl) Delete(_ context.Context, key string) error {
if !i.store.Delete(key) {
return fmt.Errorf("%w: %q", store.ErrNotFound, key)
}
return nil
}
func (i *impl) Get(_ context.Context, key string) ([]byte, error) {
result, ok := i.store.Get(key)
if !ok {
return nil, fmt.Errorf("%w: %q", store.ErrNotFound, key)
}
return result, nil
}
func (i *impl) Set(_ context.Context, key string, value []byte, expiry time.Duration) error {
i.store.Set(key, value, expiry)
return nil
}
func (i *impl) cleanupThread(ctx context.Context) {
t := time.NewTicker(5 * time.Minute)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
i.store.Cleanup()
}
}
}
// New creates a simple in-memory store. This will not scale to multiple Anubis instances.
func New(ctx context.Context) store.Interface {
result := &impl{
store: decaymap.New[string, []byte](),
}
go result.cleanupThread(ctx)
return result
}

View file

@ -0,0 +1,11 @@
package memory
import (
"testing"
"github.com/TecharoHQ/anubis/lib/store/storetest"
)
func TestImpl(t *testing.T) {
storetest.Common(t, factory{}, nil)
}

43
lib/store/registry.go Normal file
View file

@ -0,0 +1,43 @@
package store
import (
"context"
"encoding/json"
"sort"
"sync"
)
var (
registry map[string]Factory = map[string]Factory{}
regLock sync.RWMutex
)
type Factory interface {
Build(ctx context.Context, config json.RawMessage) (Interface, error)
Valid(config json.RawMessage) error
}
func Register(name string, impl Factory) {
regLock.Lock()
defer regLock.Unlock()
registry[name] = impl
}
func Get(name string) (Factory, bool) {
regLock.RLock()
defer regLock.RUnlock()
result, ok := registry[name]
return result, ok
}
func Methods() []string {
regLock.RLock()
defer regLock.RUnlock()
var result []string
for method := range registry {
result = append(result, method)
}
sort.Strings(result)
return result
}

View file

@ -0,0 +1,92 @@
package storetest
import (
"bytes"
"encoding/json"
"errors"
"testing"
"time"
"github.com/TecharoHQ/anubis/lib/store"
)
func Common(t *testing.T, f store.Factory, config json.RawMessage) {
if err := f.Valid(config); err != nil {
t.Fatal(err)
}
s, err := f.Build(t.Context(), config)
if err != nil {
t.Fatal(err)
}
for _, tt := range []struct {
name string
doer func(t *testing.T, s store.Interface) error
err error
}{
{
name: "basic get set delete",
doer: func(t *testing.T, s store.Interface) error {
if _, err := s.Get(t.Context(), t.Name()); !errors.Is(err, store.ErrNotFound) {
t.Errorf("wanted %s to not exist in store but it exists anyways", t.Name())
}
if err := s.Set(t.Context(), t.Name(), []byte(t.Name()), 5*time.Minute); err != nil {
return err
}
val, err := s.Get(t.Context(), t.Name())
if errors.Is(err, store.ErrNotFound) {
t.Errorf("wanted %s to exist in store but it does not", t.Name())
}
if !bytes.Equal(val, []byte(t.Name())) {
t.Logf("want: %q", t.Name())
t.Logf("got: %q", string(val))
t.Error("wrong value returned")
}
if err := s.Delete(t.Context(), t.Name()); err != nil {
return err
}
if _, err := s.Get(t.Context(), t.Name()); !errors.Is(err, store.ErrNotFound) {
t.Error("wanted test to not exist in store but it exists anyways")
}
if err := s.Delete(t.Context(), t.Name()); err == nil {
t.Errorf("key %q does not exist and Delete did not return non-nil", t.Name())
}
return nil
},
},
{
name: "expires",
doer: func(t *testing.T, s store.Interface) error {
if err := s.Set(t.Context(), t.Name(), []byte(t.Name()), 150*time.Millisecond); err != nil {
return err
}
//nosleep:bypass XXX(Xe): use Go's time faking thing in Go 1.25 when that is released.
time.Sleep(155 * time.Millisecond)
if _, err := s.Get(t.Context(), t.Name()); !errors.Is(err, store.ErrNotFound) {
t.Errorf("wanted %s to not exist in store but it exists anyways", t.Name())
}
return nil
},
},
} {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if err := tt.doer(t, s); !errors.Is(err, tt.err) {
t.Logf("want: %v", tt.err)
t.Logf("got: %v", err)
t.Error("wrong error")
}
})
}
}

View file

@ -0,0 +1,84 @@
package valkey
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/TecharoHQ/anubis/lib/store"
valkey "github.com/redis/go-redis/v9"
)
var (
ErrNoURL = errors.New("valkey.Config: no URL defined")
ErrBadURL = errors.New("valkey.Config: URL is invalid")
)
func init() {
store.Register("valkey", Factory{})
}
type Factory struct{}
func (Factory) Build(ctx context.Context, data json.RawMessage) (store.Interface, error) {
var config Config
if err := json.Unmarshal([]byte(data), &config); err != nil {
return nil, fmt.Errorf("%w: %w", store.ErrBadConfig, err)
}
if err := config.Valid(); err != nil {
return nil, fmt.Errorf("%w: %w", store.ErrBadConfig, err)
}
opts, err := valkey.ParseURL(config.URL)
if err != nil {
return nil, fmt.Errorf("%w: %w", store.ErrBadConfig, err)
}
rdb := valkey.NewClient(opts)
if _, err := rdb.Ping(ctx).Result(); err != nil {
return nil, fmt.Errorf("can't ping valkey instance: %w", err)
}
return &Store{
rdb: rdb,
}, nil
}
func (Factory) Valid(data json.RawMessage) error {
var config Config
if err := json.Unmarshal([]byte(data), &config); err != nil {
return fmt.Errorf("%w: %w", store.ErrBadConfig, err)
}
if err := config.Valid(); err != nil {
return fmt.Errorf("%w: %w", store.ErrBadConfig, err)
}
return nil
}
type Config struct {
URL string `json:"url"`
}
func (c Config) Valid() error {
var errs []error
if c.URL == "" {
errs = append(errs, ErrNoURL)
}
if _, err := valkey.ParseURL(c.URL); err != nil {
errs = append(errs, ErrBadURL)
}
if len(errs) != 0 {
return fmt.Errorf("valkey.Config: invalid config: %w", errors.Join(errs...))
}
return nil
}

View file

@ -0,0 +1,49 @@
package valkey
import (
"context"
"fmt"
"time"
"github.com/TecharoHQ/anubis/lib/store"
valkey "github.com/redis/go-redis/v9"
)
type Store struct {
rdb *valkey.Client
}
func (s *Store) Delete(ctx context.Context, key string) error {
n, err := s.rdb.Del(ctx, key).Result()
if err != nil {
return fmt.Errorf("can't delete from valkey: %w", err)
}
switch n {
case 0:
return fmt.Errorf("%w: %d key(s) deleted", store.ErrNotFound, n)
default:
return nil
}
}
func (s *Store) Get(ctx context.Context, key string) ([]byte, error) {
result, err := s.rdb.Get(ctx, key).Result()
if err != nil {
if valkey.HasErrorPrefix(err, "redis: nil") {
return nil, fmt.Errorf("%w: %w", store.ErrNotFound, err)
}
return nil, fmt.Errorf("can't fetch from valkey: %w", err)
}
return []byte(result), nil
}
func (s *Store) Set(ctx context.Context, key string, value []byte, expiry time.Duration) error {
if _, err := s.rdb.Set(ctx, key, string(value), expiry).Result(); err != nil {
return fmt.Errorf("can't set %q in valkey: %w", key, err)
}
return nil
}

View file

@ -0,0 +1,53 @@
package valkey
import (
"encoding/json"
"fmt"
"os"
"testing"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/store/storetest"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/wait"
)
func init() {
internal.UnbreakDocker()
}
func TestImpl(t *testing.T) {
if os.Getenv("DONT_USE_NETWORK") != "" {
t.Skip("test requires network egress")
return
}
testcontainers.SkipIfProviderIsNotHealthy(t)
req := testcontainers.ContainerRequest{
Image: "valkey/valkey:8",
WaitingFor: wait.ForLog("Ready to accept connections"),
}
valkeyC, err := testcontainers.GenericContainer(t.Context(), testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
})
testcontainers.CleanupContainer(t, valkeyC)
if err != nil {
t.Fatal(err)
}
containerIP, err := valkeyC.ContainerIP(t.Context())
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(Config{
URL: fmt.Sprintf("redis://%s:6379/0", containerIP),
})
if err != nil {
t.Fatal(err)
}
storetest.Common(t, Factory{}, json.RawMessage(data))
}