feat: writing logs to the filesystem with rotation support (#1299)

* refactor: move lib/policy/config to lib/config

Signed-off-by: Xe Iaso <me@xeiaso.net>

* refactor: don't set global loggers anymore

Ref #864

You were right @kotx, it is a bad idea to set the global logger
instance.

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(config): add log sink support

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: update spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(test): go mod tidy

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: update spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs(admin/policies): add logging block documentation

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs: update CHANGELOG

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(cmd/anubis): revert this change, it's meant to be its own PR

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: go mod tidy

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test: add file logging smoke test

Assisted-by: GLM 4.6 via Claude Code
Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix: don't expose the old log file time format string

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
This commit is contained in:
Xe Iaso 2025-11-21 11:46:00 -05:00 committed by GitHub
parent a709a2b2da
commit f032d5d0ac
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
118 changed files with 789 additions and 65 deletions

View file

@ -27,10 +27,10 @@ import (
"github.com/TecharoHQ/anubis/internal/dnsbl"
"github.com/TecharoHQ/anubis/internal/ogtags"
"github.com/TecharoHQ/anubis/lib/challenge"
"github.com/TecharoHQ/anubis/lib/config"
"github.com/TecharoHQ/anubis/lib/localization"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/store"
// challenge implementations

View file

@ -20,8 +20,8 @@ import (
"github.com/TecharoHQ/anubis/data"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/challenge"
"github.com/TecharoHQ/anubis/lib/config"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/store"
"github.com/TecharoHQ/anubis/lib/thoth/thothmock"
)
@ -58,7 +58,7 @@ func loadPolicies(t *testing.T, fname string, difficulty int) *policy.ParsedConf
t.Logf("loading policy file: %s", fname)
anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, difficulty)
anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, difficulty, "info")
if err != nil {
t.Fatal(err)
}
@ -250,7 +250,7 @@ func TestLoadPolicies(t *testing.T) {
}
defer fin.Close()
if _, err := policy.ParseConfig(t.Context(), fin, fname, 4); err != nil {
if _, err := policy.ParseConfig(t.Context(), fin, fname, 4, "info"); err != nil {
t.Fatal(err)
}
})

View file

@ -6,8 +6,8 @@ import (
"sort"
"sync"
"github.com/TecharoHQ/anubis/lib/config"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/store"
"github.com/a-h/templ"
)

View file

@ -8,8 +8,8 @@ import (
"testing"
"github.com/TecharoHQ/anubis/lib/challenge"
"github.com/TecharoHQ/anubis/lib/config"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/config"
)
func mkRequest(t *testing.T, values map[string]string) *http.Request {

View file

@ -18,9 +18,9 @@ import (
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/internal/ogtags"
"github.com/TecharoHQ/anubis/lib/challenge"
"github.com/TecharoHQ/anubis/lib/config"
"github.com/TecharoHQ/anubis/lib/localization"
"github.com/TecharoHQ/anubis/lib/policy"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/web"
"github.com/TecharoHQ/anubis/xess"
"github.com/a-h/templ"
@ -48,12 +48,13 @@ type Options struct {
CookieSecure bool
CookieSameSite http.SameSite
Logger *slog.Logger
LogLevel string
PublicUrl string
JWTRestrictionHeader string
DifficultyInJWT bool
}
func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int) (*policy.ParsedConfig, error) {
func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int, logLevel string) (*policy.ParsedConfig, error) {
var fin io.ReadCloser
var err error
@ -77,7 +78,7 @@ func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty
}
}(fin)
anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty)
anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty, logLevel)
if err != nil {
return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err)
}

View file

@ -332,6 +332,7 @@ type fileConfig struct {
Thresholds []Threshold `json:"thresholds"`
StatusCodes StatusCodes `json:"status_codes"`
DNSBL bool `json:"dnsbl"`
Logging *Logging `json:"logging"`
}
func (c *fileConfig) Valid() error {
@ -363,6 +364,10 @@ func (c *fileConfig) Valid() error {
}
}
if err := c.Logging.Valid(); err != nil {
errs = append(errs, err)
}
if c.Store != nil {
if err := c.Store.Valid(); err != nil {
errs = append(errs, err)
@ -385,6 +390,7 @@ func Load(fin io.Reader, fname string) (*Config, error) {
Store: &Store{
Backend: "memory",
},
Logging: (Logging{}).Default(),
}
if err := yaml.NewYAMLToJSONDecoder(fin).Decode(&c); err != nil {
@ -404,6 +410,7 @@ func Load(fin io.Reader, fname string) (*Config, error) {
},
StatusCodes: c.StatusCodes,
Store: c.Store,
Logging: c.Logging,
}
if c.OpenGraph.TimeToLive != "" {
@ -469,6 +476,7 @@ type Config struct {
Bots []BotConfig
Thresholds []Threshold
StatusCodes StatusCodes
Logging *Logging
DNSBL bool
}

View file

@ -8,7 +8,7 @@ import (
"testing"
"github.com/TecharoHQ/anubis/data"
. "github.com/TecharoHQ/anubis/lib/policy/config"
. "github.com/TecharoHQ/anubis/lib/config"
)
func p[V any](v V) *V { return &v }

124
lib/config/logging.go Normal file
View file

@ -0,0 +1,124 @@
package config
import (
"errors"
"fmt"
"log/slog"
)
var (
ErrMissingLoggingFileConfig = errors.New("config.Logging: missing value parameters in logging block")
ErrInvalidLoggingSink = errors.New("config.Logging: invalid sink")
ErrInvalidLoggingFileConfig = errors.New("config.LoggingFileConfig: invalid parameters")
ErrOutOfRange = errors.New("config: error out of range")
)
type Logging struct {
Sink string `json:"sink"` // Logging sink, either "stdio" or "file"
Level *slog.Level `json:"level"` // Log level, if set supercedes the level in flags
Parameters *LoggingFileConfig `json:"parameters"` // Logging parameters, to be dynamic in the future
}
const (
LogSinkStdio = "stdio"
LogSinkFile = "file"
)
func (l *Logging) Valid() error {
var errs []error
switch l.Sink {
case LogSinkStdio:
// no validation needed
case LogSinkFile:
if l.Parameters == nil {
errs = append(errs, ErrMissingLoggingFileConfig)
}
if err := l.Parameters.Valid(); err != nil {
errs = append(errs, err)
}
default:
errs = append(errs, fmt.Errorf("%w: sink %s is unknown to me", ErrInvalidLoggingSink, l.Sink))
}
if len(errs) != 0 {
return errors.Join(errs...)
}
return nil
}
func (Logging) Default() *Logging {
return &Logging{
Sink: "stdio",
}
}
type LoggingFileConfig struct {
Filename string `json:"file"`
MaxBackups int `json:"maxBackups"`
MaxBytes int64 `json:"maxBytes"`
MaxAge int `json:"maxAge"`
Compress bool `json:"compress"`
UseLocalTime bool `json:"useLocalTime"`
}
func (lfc *LoggingFileConfig) Valid() error {
if lfc == nil {
return fmt.Errorf("logging file config is nil, why are you calling this?")
}
var errs []error
if lfc.Zero() {
errs = append(errs, ErrMissingValue)
}
if lfc.Filename == "" {
errs = append(errs, fmt.Errorf("%w: filename", ErrMissingValue))
}
if lfc.MaxBackups < 0 {
errs = append(errs, fmt.Errorf("%w: max backup count %d is not greater than or equal to zero", ErrOutOfRange, lfc.MaxBackups))
}
if lfc.MaxAge < 0 {
errs = append(errs, fmt.Errorf("%w: max backup count %d is not greater than or equal to zero", ErrOutOfRange, lfc.MaxAge))
}
if len(errs) != 0 {
errs = append([]error{ErrInvalidLoggingFileConfig}, errs...)
return errors.Join(errs...)
}
return nil
}
func (lfc LoggingFileConfig) Zero() bool {
for _, cond := range []bool{
lfc.Filename != "",
lfc.MaxBackups != 0,
lfc.MaxBytes != 0,
lfc.MaxAge != 0,
lfc.Compress,
lfc.UseLocalTime,
} {
if cond {
return false
}
}
return true
}
func (LoggingFileConfig) Default() *LoggingFileConfig {
return &LoggingFileConfig{
Filename: "./var/anubis.log",
MaxBackups: 3,
MaxBytes: 104857600, // 100 Mi
MaxAge: 7, // 7 days
Compress: true,
UseLocalTime: false,
}
}

103
lib/config/logging_test.go Normal file
View file

@ -0,0 +1,103 @@
package config
import (
"errors"
"testing"
)
func TestLoggingValid(t *testing.T) {
for _, tt := range []struct {
name string
input *Logging
want error
}{
{
name: "simple happy",
input: (Logging{}).Default(),
},
{
name: "default file config",
input: &Logging{
Sink: LogSinkFile,
Parameters: (&LoggingFileConfig{}).Default(),
},
},
{
name: "invalid sink",
input: &Logging{
Sink: "taco invalid",
},
want: ErrInvalidLoggingSink,
},
{
name: "missing parameters",
input: &Logging{
Sink: LogSinkFile,
},
want: ErrMissingLoggingFileConfig,
},
{
name: "invalid parameters",
input: &Logging{
Sink: LogSinkFile,
Parameters: &LoggingFileConfig{},
},
want: ErrInvalidLoggingFileConfig,
},
{
name: "file sink with no filename",
input: &Logging{
Sink: LogSinkFile,
Parameters: &LoggingFileConfig{
Filename: "",
MaxBackups: 3,
MaxBytes: 104857600, // 100 Mi
MaxAge: 7, // 7 days
Compress: true,
UseLocalTime: false,
},
},
want: ErrMissingValue,
},
{
name: "file sink with negative max backups",
input: &Logging{
Sink: LogSinkFile,
Parameters: &LoggingFileConfig{
Filename: "./var/anubis.log",
MaxBackups: -3,
MaxBytes: 104857600, // 100 Mi
MaxAge: 7, // 7 days
Compress: true,
UseLocalTime: false,
},
},
want: ErrOutOfRange,
},
{
name: "file sink with negative max age",
input: &Logging{
Sink: LogSinkFile,
Parameters: &LoggingFileConfig{
Filename: "./var/anubis.log",
MaxBackups: 3,
MaxBytes: 104857600, // 100 Mi
MaxAge: -7, // 7 days
Compress: true,
UseLocalTime: false,
},
},
want: ErrOutOfRange,
},
} {
t.Run(tt.name, func(t *testing.T) {
err := tt.input.Valid()
if !errors.Is(err, tt.want) {
t.Logf("wanted error: %v", tt.want)
t.Logf(" got error: %v", err)
t.Fatal("got wrong error")
}
})
}
}

View file

@ -5,7 +5,7 @@ import (
"errors"
"testing"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/config"
"github.com/TecharoHQ/anubis/lib/store/bbolt"
"github.com/TecharoHQ/anubis/lib/store/valkey"
)

View file

@ -0,0 +1,2 @@
logging:
sink: "nope"

View file

@ -0,0 +1,2 @@
logging:
sink: "file"

View file

@ -0,0 +1,15 @@
bots:
- name: simple
action: CHALLENGE
user_agent_regex: Mozilla
logs:
sink: "file"
parameters:
file: "/var/log/botstopper/default.log"
maxBackups: 3 # keep at least 3 old copies
maxBytes: 67108864 # each file can have up to 64 MB of logs
maxAge: 7 # rotate files out every n days
oldFileTimeFormat: 2006-01-02T15-04-05 # RFC 3339-ish
compress: true
useLocalTime: false # timezone for rotated files is UTC

View file

@ -0,0 +1,7 @@
bots:
- name: simple
action: CHALLENGE
user_agent_regex: Mozilla
logging:
sink: "stdio"

View file

@ -12,13 +12,13 @@ import (
)
func TestInvalidChallengeMethod(t *testing.T) {
if _, err := LoadPoliciesOrDefault(t.Context(), "testdata/invalid-challenge-method.yaml", 4); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) {
if _, err := LoadPoliciesOrDefault(t.Context(), "testdata/invalid-challenge-method.yaml", 4, "info"); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) {
t.Fatalf("wanted error %v but got %v", policy.ErrChallengeRuleHasWrongAlgorithm, err)
}
}
func TestBadConfigs(t *testing.T) {
finfos, err := os.ReadDir("policy/config/testdata/bad")
finfos, err := os.ReadDir("config/testdata/bad")
if err != nil {
t.Fatal(err)
}
@ -26,7 +26,7 @@ func TestBadConfigs(t *testing.T) {
for _, st := range finfos {
st := st
t.Run(st.Name(), func(t *testing.T) {
if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("policy", "config", "testdata", "bad", st.Name()), anubis.DefaultDifficulty); err == nil {
if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "bad", st.Name()), anubis.DefaultDifficulty, "info"); err == nil {
t.Fatal(err)
} else {
t.Log(err)
@ -36,7 +36,7 @@ func TestBadConfigs(t *testing.T) {
}
func TestGoodConfigs(t *testing.T) {
finfos, err := os.ReadDir("policy/config/testdata/good")
finfos, err := os.ReadDir("config/testdata/good")
if err != nil {
t.Fatal(err)
}
@ -46,13 +46,13 @@ func TestGoodConfigs(t *testing.T) {
t.Run(st.Name(), func(t *testing.T) {
t.Run("with-thoth", func(t *testing.T) {
ctx := thothmock.WithMockThoth(t)
if _, err := LoadPoliciesOrDefault(ctx, filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err != nil {
if _, err := LoadPoliciesOrDefault(ctx, filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info"); err != nil {
t.Fatal(err)
}
})
t.Run("without-thoth", func(t *testing.T) {
if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err != nil {
if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("config", "testdata", "good", st.Name()), anubis.DefaultDifficulty, "info"); err != nil {
t.Fatal(err)
}
})

View file

@ -4,8 +4,8 @@ import (
"fmt"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/config"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/policy/config"
)
type Bot struct {

View file

@ -5,7 +5,7 @@ import (
"net/http"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/config"
"github.com/TecharoHQ/anubis/lib/policy/expressions"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common/types"

View file

@ -3,7 +3,7 @@ package policy
import (
"log/slog"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/config"
)
type CheckResult struct {

View file

@ -6,12 +6,16 @@ import (
"fmt"
"io"
"log/slog"
"os"
"sync/atomic"
"time"
"github.com/TecharoHQ/anubis/internal"
"github.com/TecharoHQ/anubis/lib/config"
"github.com/TecharoHQ/anubis/lib/policy/checker"
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/store"
"github.com/TecharoHQ/anubis/lib/thoth"
"github.com/fahedouch/go-logrotate"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -38,6 +42,7 @@ type ParsedConfig struct {
StatusCodes config.StatusCodes
DefaultDifficulty int
DNSBL bool
Logger *slog.Logger
}
func newParsedConfig(orig *config.Config) *ParsedConfig {
@ -48,7 +53,7 @@ func newParsedConfig(orig *config.Config) *ParsedConfig {
}
}
func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) {
func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDifficulty int, logLevel string) (*ParsedConfig, error) {
c, err := config.Load(fin, fname)
if err != nil {
return nil, err
@ -202,6 +207,27 @@ func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDiffic
validationErrs = append(validationErrs, config.ErrUnknownStoreBackend)
}
if c.Logging.Level != nil {
logLevel = c.Logging.Level.String()
}
switch c.Logging.Sink {
case config.LogSinkStdio:
result.Logger = internal.InitSlog(logLevel, os.Stderr)
case config.LogSinkFile:
out := &logrotate.Logger{
Filename: c.Logging.Parameters.Filename,
FilenameTimeFormat: time.RFC3339,
MaxBytes: c.Logging.Parameters.MaxBytes,
MaxAge: c.Logging.Parameters.MaxAge,
MaxBackups: c.Logging.Parameters.MaxBackups,
LocalTime: c.Logging.Parameters.UseLocalTime,
Compress: c.Logging.Parameters.Compress,
}
result.Logger = internal.InitSlog(logLevel, out)
}
if len(validationErrs) > 0 {
return nil, fmt.Errorf("errors validating policy config JSON %s: %w", fname, errors.Join(validationErrs...))
}

View file

@ -19,14 +19,14 @@ func TestDefaultPolicyMustParse(t *testing.T) {
}
defer fin.Close()
if _, err := ParseConfig(ctx, fin, "botPolicies.yaml", anubis.DefaultDifficulty); err != nil {
if _, err := ParseConfig(ctx, fin, "botPolicies.yaml", anubis.DefaultDifficulty, "info"); err != nil {
t.Fatalf("can't parse config: %v", err)
}
}
func TestGoodConfigs(t *testing.T) {
finfos, err := os.ReadDir("config/testdata/good")
finfos, err := os.ReadDir("../config/testdata/good")
if err != nil {
t.Fatal(err)
}
@ -35,26 +35,26 @@ func TestGoodConfigs(t *testing.T) {
st := st
t.Run(st.Name(), func(t *testing.T) {
t.Run("with-thoth", func(t *testing.T) {
fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name()))
fin, err := os.Open(filepath.Join("..", "config", "testdata", "good", st.Name()))
if err != nil {
t.Fatal(err)
}
defer fin.Close()
ctx := thothmock.WithMockThoth(t)
if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty); err != nil {
if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info"); err != nil {
t.Fatal(err)
}
})
t.Run("without-thoth", func(t *testing.T) {
fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name()))
fin, err := os.Open(filepath.Join("..", "config", "testdata", "good", st.Name()))
if err != nil {
t.Fatal(err)
}
defer fin.Close()
if _, err := ParseConfig(t.Context(), fin, fin.Name(), anubis.DefaultDifficulty); err != nil {
if _, err := ParseConfig(t.Context(), fin, fin.Name(), anubis.DefaultDifficulty, "info"); err != nil {
t.Fatal(err)
}
})
@ -65,7 +65,7 @@ func TestGoodConfigs(t *testing.T) {
func TestBadConfigs(t *testing.T) {
ctx := thothmock.WithMockThoth(t)
finfos, err := os.ReadDir("config/testdata/bad")
finfos, err := os.ReadDir("../config/testdata/bad")
if err != nil {
t.Fatal(err)
}
@ -73,13 +73,13 @@ func TestBadConfigs(t *testing.T) {
for _, st := range finfos {
st := st
t.Run(st.Name(), func(t *testing.T) {
fin, err := os.Open(filepath.Join("config", "testdata", "bad", st.Name()))
fin, err := os.Open(filepath.Join("..", "config", "testdata", "bad", st.Name()))
if err != nil {
t.Fatal(err)
}
defer fin.Close()
if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty); err == nil {
if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty, "info"); err == nil {
t.Fatal(err)
} else {
t.Log(err)

View file

@ -1,7 +1,7 @@
package policy
import (
"github.com/TecharoHQ/anubis/lib/policy/config"
"github.com/TecharoHQ/anubis/lib/config"
"github.com/TecharoHQ/anubis/lib/policy/expressions"
"github.com/google/cel-go/cel"
)