sophie/nuke
1
0
Fork 0
Code Packages 1 Activity Actions

feat: writing logs to the filesystem with rotation support (#1299)

Browse source 
* refactor: move lib/policy/config to lib/config

Signed-off-by: Xe Iaso <me@xeiaso.net>

* refactor: don't set global loggers anymore

Ref #864

You were right @kotx, it is a bad idea to set the global logger
instance.

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(config): add log sink support

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: update spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(test): go mod tidy

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: update spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs(admin/policies): add logging block documentation

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs: update CHANGELOG

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(cmd/anubis): revert this change, it's meant to be its own PR

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: go mod tidy

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test: add file logging smoke test

Assisted-by: GLM 4.6 via Claude Code
Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix: don't expose the old log file time format string

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
This commit is contained in:
Xe Iaso 2025-11-21 11:46:00 -05:00 • committed by GitHub
parent a709a2b2da
commit f032d5d0ac
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
118 changed files with 789 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

21
lib/config/testdata/bad/badregexes.json vendored Normal file
View file

@ -0,0 +1,21 @@
{
"bots": [
{
"name": "path-bad",
"path_regex": "a(b",
"action": "DENY"
},
{
"name": "user-agent-bad",
"user_agent_regex": "a(b",
"action": "DENY"
},
{
"name": "headers-bad",
"headers": {
"Accept-Encoding": "a(b"
},
"action": "DENY"
}
]
}

7
lib/config/testdata/bad/badregexes.yaml vendored Normal file
View file

@ -0,0 +1,7 @@
bots:
- name: path-bad
path_regex: "a(b"
action: DENY
- name: user-agent-bad
user_agent_regex: "a(b"
action: DENY

10
lib/config/testdata/bad/import_and_bot.json vendored Normal file
View file

@ -0,0 +1,10 @@
{
"bots": [
{
"import": "(data)/bots/ai-catchall.yaml",
"name": "generic-browser",
"user_agent_regex": "Mozilla|Opera\n",
"action": "CHALLENGE"
}
]
}

6
lib/config/testdata/bad/import_and_bot.yaml vendored Normal file
View file

@ -0,0 +1,6 @@
bots:
- import: (data)/bots/ai-catchall.yaml
name: generic-browser
user_agent_regex: >
Mozilla|Opera
action: CHALLENGE

7
lib/config/testdata/bad/import_invalid_file.json vendored Normal file
View file

@ -0,0 +1,7 @@
{
"bots": [
{
"import": "(data)/does-not-exist-fake-file.yaml"
}
]
}

2
lib/config/testdata/bad/import_invalid_file.yaml vendored Normal file
View file

@ -0,0 +1,2 @@
bots:
- import: (data)/does-not-exist-fake-file.yaml

11
lib/config/testdata/bad/impressum-no-footer.yaml vendored Normal file
View file

@ -0,0 +1,11 @@
bots:
- name: simple-weight-adjust
action: WEIGH
user_agent_regex: Mozilla
weight:
adjust: 5
impressum:
page:
title: Test
body: <p>This is a test</p>

10
lib/config/testdata/bad/impressum-no-page-contents.yaml vendored Normal file
View file

@ -0,0 +1,10 @@
bots:
- name: simple-weight-adjust
action: WEIGH
user_agent_regex: Mozilla
weight:
adjust: 5
impressum:
footer: "Hi there these are WORDS on the INTERNET."
page: {}

5
lib/config/testdata/bad/invalid.json vendored Normal file
View file

@ -0,0 +1,5 @@
{
"bots": [
{}
]
}

1
lib/config/testdata/bad/invalid.yaml vendored Normal file
View file

@ -0,0 +1 @@
bots: []

2
lib/config/testdata/bad/logging-invalid-sink.yaml vendored Normal file
View file

@ -0,0 +1,2 @@
logging:
sink: "nope"

2
lib/config/testdata/bad/logging-no-parameters.yaml vendored Normal file
View file

@ -0,0 +1,2 @@
logging:
sink: "file"

17
lib/config/testdata/bad/multiple_expression_types.json vendored Normal file
View file

@ -0,0 +1,17 @@
{
"bots": [
{
"name": "multiple-expression-types",
"action": "ALLOW",
"expression": {
"all": [
"userAgent.startsWith(\"git/\") || userAgent.contains(\"libgit\")",
"\"Git-Protocol\" in headers && headers[\"Git-Protocol\"] == \"version=2\"\n"
],
"any": [
"userAgent.startsWith(\"evilbot/\")"
]
}
}
]
}

10
lib/config/testdata/bad/multiple_expression_types.yaml vendored Normal file
View file

@ -0,0 +1,10 @@
bots:
- name: multiple-expression-types
action: ALLOW
expression:
all:
- userAgent.startsWith("git/") || userAgent.contains("libgit")
- >
"Git-Protocol" in headers && headers["Git-Protocol"] == "version=2"
any:
- userAgent.startsWith("evilbot/")

1
lib/config/testdata/bad/nobots.json vendored Normal file
View file

@ -0,0 +1 @@
{}

1
lib/config/testdata/bad/nobots.yaml vendored Normal file
View file

@ -0,0 +1 @@
{}

12
lib/config/testdata/bad/opengraph_bad_ttl.yaml vendored Normal file
View file

@ -0,0 +1,12 @@
bots:
- name: everything
user_agent_regex: .*
action: DENY
openGraph:
enabled: true
considerHost: false
ttl: taco
default:
"og:title": "Xe's magic land of fun"
"og:description": "We're no strangers to love, you know the rules and so do I"

21
lib/config/testdata/bad/regex_ends_newline.json vendored Normal file
View file

@ -0,0 +1,21 @@
{
"bots": [
{
"name": "user-agent-ends-newline",
"user_agent_regex": "Mozilla\n",
"action": "CHALLENGE"
},
{
"name": "path-ends-newline",
"path_regex": "^/evil/.*$\n",
"action": "CHALLENGE"
},
{
"name": "headers-ends-newline",
"headers_regex": {
"CF-Worker": ".*\n"
},
"action": "CHALLENGE"
}
]
}

17
lib/config/testdata/bad/regex_ends_newline.yaml vendored Normal file
View file

@ -0,0 +1,17 @@
bots:
- name: user-agent-ends-newline
# Subtle bug: this ends with a newline
user_agent_regex: >
Mozilla
action: CHALLENGE
- name: path-ends-newline
# Subtle bug: this ends with a newline
path_regex: >
^/evil/.*$
action: CHALLENGE
- name: headers-ends-newline
# Subtle bug: this ends with a newline
headers_regex:
CF-Worker: >
.*
action: CHALLENGE

13
lib/config/testdata/bad/status-codes-0.json vendored Normal file
View file

@ -0,0 +1,13 @@
{
"bots": [
{
"name": "everything",
"user_agent_regex": ".*",
"action": "DENY"
}
],
"status_codes": {
"CHALLENGE": 0,
"DENY": 0
}
}

8
lib/config/testdata/bad/status-codes-0.yaml vendored Normal file
View file

@ -0,0 +1,8 @@
bots:
- name: everything
user_agent_regex: .*
action: DENY
status_codes:
CHALLENGE: 0
DENY: 0

11
lib/config/testdata/bad/threshold-challenge-without-challenge.yaml vendored Normal file
View file

@ -0,0 +1,11 @@
bots:
- name: simple-weight-adjust
action: WEIGH
user_agent_regex: Mozilla
weight:
adjust: 5
thresholds:
- name: extreme-suspicion
expression: "true"
action: WEIGH

15
lib/config/testdata/bad/thresholds.yaml vendored Normal file
View file

@ -0,0 +1,15 @@
bots:
- name: simple-weight-adjust
action: WEIGH
user_agent_regex: Mozilla
weight:
adjust: 5
thresholds:
- name: extreme-suspicion
expression: "true"
action: WEIGH
challenge:
algorithm: fast
difficulty: 4
report_as: 4

1
lib/config/testdata/bad/unparseable.json vendored Normal file
View file

@ -0,0 +1 @@
}

1
lib/config/testdata/bad/unparseable.yaml vendored Normal file
View file

@ -0,0 +1 @@
}

12
lib/config/testdata/good/allow_everyone.json vendored Normal file
View file

@ -0,0 +1,12 @@
{
"bots": [
{
"name": "everyones-invited",
"remote_addresses": [
"0.0.0.0/0",
"::/0"
],
"action": "ALLOW"
}
]
}

6
lib/config/testdata/good/allow_everyone.yaml vendored Normal file
View file

@ -0,0 +1,6 @@
bots:
- name: everyones-invited
remote_addresses:
- "0.0.0.0/0"
- "::/0"
action: ALLOW

12
lib/config/testdata/good/block_cf_workers.json vendored Normal file
View file

@ -0,0 +1,12 @@
{
"bots": [
{
"name": "Cloudflare Workers",
"headers_regex": {
"CF-Worker": ".*"
},
"action": "DENY"
}
],
"dnsbl": false
}

5
lib/config/testdata/good/block_cf_workers.yaml vendored Normal file
View file

@ -0,0 +1,5 @@
bots:
- name: cloudflare-workers
headers_regex:
CF-Worker: .*
action: DENY

6
lib/config/testdata/good/challenge_cloudflare.yaml vendored Normal file
View file

@ -0,0 +1,6 @@
bots:
- name: challenge-cloudflare
action: CHALLENGE
asns:
match:
- 13335 # Cloudflare

9
lib/config/testdata/good/challengemozilla.json vendored Normal file
View file

@ -0,0 +1,9 @@
{
"bots": [
{
"name": "generic-browser",
"user_agent_regex": "Mozilla",
"action": "CHALLENGE"
}
]
}

4
lib/config/testdata/good/challengemozilla.yaml vendored Normal file
View file

@ -0,0 +1,4 @@
bots:
- name: generic-browser
user_agent_regex: Mozilla
action: CHALLENGE

8
lib/config/testdata/good/entropy.yaml vendored Normal file
View file

@ -0,0 +1,8 @@
bots:
- name: total-randomness
action: ALLOW
expression:
all:
- '"Accept" in headers'
- headers["Accept"].contains("text/html")
- randInt(1) == 0

10
lib/config/testdata/good/everything_blocked.json vendored Normal file
View file

@ -0,0 +1,10 @@
{
"bots": [
{
"name": "everything",
"user_agent_regex": ".*",
"action": "DENY"
}
],
"dnsbl": false
}

4
lib/config/testdata/good/everything_blocked.yaml vendored Normal file
View file

@ -0,0 +1,4 @@
bots:
- name: everything
user_agent_regex: .*
action: DENY

6
lib/config/testdata/good/geoip_us.yaml vendored Normal file
View file

@ -0,0 +1,6 @@
bots:
- name: compute-tarrif-us
action: CHALLENGE
geoip:
countries:
- US

14
lib/config/testdata/good/git_client.json vendored Normal file
View file

@ -0,0 +1,14 @@
{
"bots": [
{
"name": "allow-git-clients",
"action": "ALLOW",
"expression": {
"all": [
"userAgent.startsWith(\"git/\") || userAgent.contains(\"libgit\")",
"\"Git-Protocol\" in headers && headers[\"Git-Protocol\"] == \"version=2\""
]
}
}
]
}

8
lib/config/testdata/good/git_client.yaml vendored Normal file
View file

@ -0,0 +1,8 @@
bots:
- name: allow-git-clients
action: ALLOW
expression:
all:
- userAgent.startsWith("git/") || userAgent.contains("libgit")
- >
"Git-Protocol" in headers && headers["Git-Protocol"] == "version=2"

7
lib/config/testdata/good/import_filesystem.json vendored Normal file
View file

@ -0,0 +1,7 @@
{
"bots": [
{
"import": "./testdata/hack-test.json"
}
]
}

2
lib/config/testdata/good/import_filesystem.yaml vendored Normal file
View file

@ -0,0 +1,2 @@
bots:
- import: ./testdata/hack-test.yaml

7
lib/config/testdata/good/import_keep_internet_working.json vendored Normal file
View file

@ -0,0 +1,7 @@
{
"bots": [
{
"import": "(data)/common/keep-internet-working.yaml"
}
]
}

2
lib/config/testdata/good/import_keep_internet_working.yaml vendored Normal file
View file

@ -0,0 +1,2 @@
bots:
- import: (data)/common/keep-internet-working.yaml

10
lib/config/testdata/good/impressum.yaml vendored Normal file
View file

@ -0,0 +1,10 @@
bots:
- name: simple
action: CHALLENGE
user_agent_regex: Mozilla
impressum:
footer: "Hi these are WORDS on the INTERNET."
page:
title: Test
body: <p>This is a test</p>

15
lib/config/testdata/good/logging-file.yaml vendored Normal file
View file

@ -0,0 +1,15 @@
bots:
- name: simple
action: CHALLENGE
user_agent_regex: Mozilla
logs:
sink: "file"
parameters:
file: "/var/log/botstopper/default.log"
maxBackups: 3 # keep at least 3 old copies
maxBytes: 67108864 # each file can have up to 64 MB of logs
maxAge: 7 # rotate files out every n days
oldFileTimeFormat: 2006-01-02T15-04-05 # RFC 3339-ish
compress: true
useLocalTime: false # timezone for rotated files is UTC

7
lib/config/testdata/good/logging-stdio.yaml vendored Normal file
View file

@ -0,0 +1,7 @@
bots:
- name: simple
action: CHALLENGE
user_agent_regex: Mozilla
logging:
sink: "stdio"

8
lib/config/testdata/good/no-thresholds.yaml vendored Normal file
View file

@ -0,0 +1,8 @@
bots:
- name: simple-weight-adjust
action: WEIGH
user_agent_regex: Mozilla
weight:
adjust: 5
thresholds: []

79
lib/config/testdata/good/old_xesite.json vendored Normal file
View file

@ -0,0 +1,79 @@
{
"bots": [
{
"name": "amazonbot",
"user_agent_regex": "Amazonbot",
"action": "DENY"
},
{
"name": "googlebot",
"user_agent_regex": "\\+http\\:\\/\\/www\\.google\\.com/bot\\.html",
"action": "ALLOW"
},
{
"name": "bingbot",
"user_agent_regex": "\\+http\\:\\/\\/www\\.bing\\.com/bingbot\\.htm",
"action": "ALLOW"
},
{
"name": "qwantbot",
"user_agent_regex": "\\+https\\:\\/\\/help\\.qwant\\.com/bot/",
"action": "ALLOW"
},
{
"name": "discordbot",
"user_agent_regex": "Discordbot/2\\.0; \\+https\\:\\/\\/discordapp\\.com",
"action": "ALLOW"
},
{
"name": "blueskybot",
"user_agent_regex": "Bluesky Cardyb",
"action": "ALLOW"
},
{
"name": "us-artificial-intelligence-scraper",
"user_agent_regex": "\\+https\\:\\/\\/github\\.com\\/US-Artificial-Intelligence\\/scraper",
"action": "DENY"
},
{
"name": "well-known",
"path_regex": "^/.well-known/.*$",
"action": "ALLOW"
},
{
"name": "favicon",
"path_regex": "^/favicon.ico$",
"action": "ALLOW"
},
{
"name": "robots-txt",
"path_regex": "^/robots.txt$",
"action": "ALLOW"
},
{
"name": "rss-readers",
"path_regex": ".*\\.(rss|xml|atom|json)$",
"action": "ALLOW"
},
{
"name": "lightpanda",
"user_agent_regex": "^Lightpanda/.*$",
"action": "DENY"
},
{
"name": "headless-chrome",
"user_agent_regex": "HeadlessChrome",
"action": "DENY"
},
{
"name": "headless-chromium",
"user_agent_regex": "HeadlessChromium",
"action": "DENY"
},
{
"name": "generic-browser",
"user_agent_regex": "Mozilla",
"action": "CHALLENGE"
}
]
}

12
lib/config/testdata/good/opengraph_all_good.yaml vendored Normal file
View file

@ -0,0 +1,12 @@
bots:
- name: everything
user_agent_regex: .*
action: DENY
openGraph:
enabled: true
considerHost: false
ttl: 1h
default:
"og:title": "Xe's magic land of fun"
"og:description": "We're no strangers to love, you know the rules and so do I"

6
lib/config/testdata/good/simple-weight.yaml vendored Normal file
View file

@ -0,0 +1,6 @@
bots:
- name: simple-weight-adjust
action: WEIGH
user_agent_regex: Mozilla
weight:
adjust: 5

13
lib/config/testdata/good/status-codes-paranoid.json vendored Normal file
View file

@ -0,0 +1,13 @@
{
"bots": [
{
"name": "everything",
"user_agent_regex": ".*",
"action": "DENY"
}
],
"status_codes": {
"CHALLENGE": 200,
"DENY": 200
}
}

8
lib/config/testdata/good/status-codes-paranoid.yaml vendored Normal file
View file

@ -0,0 +1,8 @@
bots:
- name: everything
user_agent_regex: .*
action: DENY
status_codes:
CHALLENGE: 200
DENY: 200

13
lib/config/testdata/good/status-codes-rfc.json vendored Normal file
View file

@ -0,0 +1,13 @@
{
"bots": [
{
"name": "everything",
"user_agent_regex": ".*",
"action": "DENY"
}
],
"status_codes": {
"CHALLENGE": 403,
"DENY": 403
}
}

8
lib/config/testdata/good/status-codes-rfc.yaml vendored Normal file
View file

@ -0,0 +1,8 @@
bots:
- name: everything
user_agent_regex: .*
action: DENY
status_codes:
CHALLENGE: 403
DENY: 403

38
lib/config/testdata/good/thresholds.yaml vendored Normal file
View file

@ -0,0 +1,38 @@
bots:
- name: simple-weight-adjust
action: WEIGH
user_agent_regex: Mozilla
weight:
adjust: 5
thresholds:
- name: minimal-suspicion
expression: weight < 0
action: ALLOW
- name: mild-suspicion
expression:
all:
- weight >= 0
- weight < 10
action: CHALLENGE
challenge:
algorithm: metarefresh
difficulty: 1
report_as: 1
- name: moderate-suspicion
expression:
all:
- weight >= 10
- weight < 20
action: CHALLENGE
challenge:
algorithm: fast
difficulty: 2
report_as: 2
- name: extreme-suspicion
expression: weight >= 20
action: CHALLENGE
challenge:
algorithm: fast
difficulty: 4
report_as: 4

4
lib/config/testdata/good/weight-no-weight.yaml vendored Normal file
View file

@ -0,0 +1,4 @@
bots:
- name: weight
action: WEIGH
user_agent_regex: Mozilla

9
lib/config/testdata/hack-test.json vendored Normal file
View file

@ -0,0 +1,9 @@
[
{
"name": "ipv6-ula",
"action": "ALLOW",
"remote_addresses": [
"fc00::/7"
]
}
]

3
lib/config/testdata/hack-test.yaml vendored Normal file
View file

@ -0,0 +1,3 @@
- name: well-known
path_regex: ^/.well-known/.*$
action: ALLOW