feat: writing logs to the filesystem with rotation support (#1299)

* refactor: move lib/policy/config to lib/config

Signed-off-by: Xe Iaso <me@xeiaso.net>

* refactor: don't set global loggers anymore

Ref #864

You were right @kotx, it is a bad idea to set the global logger
instance.

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(config): add log sink support

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: update spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(test): go mod tidy

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: update spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs(admin/policies): add logging block documentation

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs: update CHANGELOG

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(cmd/anubis): revert this change, it's meant to be its own PR

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: go mod tidy

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test: add file logging smoke test

Assisted-by: GLM 4.6 via Claude Code
Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix: don't expose the old log file time format string

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>

lib/config/testdata/good/allow_everyone.json

@@ -0,0 +1,12 @@
+{
+  "bots": [
+    {
+      "name": "everyones-invited",
+      "remote_addresses": [
+        "0.0.0.0/0",
+        "::/0"
+      ],
+      "action": "ALLOW"
+    }
+  ]
+}

lib/config/testdata/good/allow_everyone.yaml

@@ -0,0 +1,6 @@
+bots:
+- name: everyones-invited
+  remote_addresses:
+    - "0.0.0.0/0"
+    - "::/0"
+  action: ALLOW

lib/config/testdata/good/block_cf_workers.json

@@ -0,0 +1,12 @@
+{
+  "bots": [
+    {
+      "name": "Cloudflare Workers",
+      "headers_regex": {
+        "CF-Worker": ".*"
+      },
+      "action": "DENY"
+    }
+  ],
+  "dnsbl": false
+}

lib/config/testdata/good/block_cf_workers.yaml

@@ -0,0 +1,5 @@
+bots:
+  - name: cloudflare-workers
+    headers_regex:
+      CF-Worker: .*
+    action: DENY

lib/config/testdata/good/challenge_cloudflare.yaml

@@ -0,0 +1,6 @@
+bots:
+  - name: challenge-cloudflare
+    action: CHALLENGE
+    asns:
+      match:
+        - 13335 # Cloudflare

lib/config/testdata/good/challengemozilla.json

@@ -0,0 +1,9 @@
+{
+  "bots": [
+    {
+      "name": "generic-browser",
+      "user_agent_regex": "Mozilla",
+      "action": "CHALLENGE"
+    }
+  ]
+}

lib/config/testdata/good/challengemozilla.yaml

@@ -0,0 +1,4 @@
+bots:
+- name: generic-browser
+  user_agent_regex: Mozilla
+  action: CHALLENGE

lib/config/testdata/good/entropy.yaml

@@ -0,0 +1,8 @@
+bots:
+  - name: total-randomness
+    action: ALLOW
+    expression:
+      all:
+        - '"Accept" in headers'
+        - headers["Accept"].contains("text/html")
+        - randInt(1) == 0

lib/config/testdata/good/everything_blocked.json

@@ -0,0 +1,10 @@
+{
+  "bots": [
+    {
+      "name": "everything",
+      "user_agent_regex": ".*",
+      "action": "DENY"
+    }
+  ],
+  "dnsbl": false
+}

lib/config/testdata/good/everything_blocked.yaml

@@ -0,0 +1,4 @@
+bots:
+- name: everything
+  user_agent_regex: .*
+  action: DENY

lib/config/testdata/good/geoip_us.yaml

@@ -0,0 +1,6 @@
+bots:
+  - name: compute-tarrif-us
+    action: CHALLENGE
+    geoip:
+      countries:
+        - US

lib/config/testdata/good/git_client.json

@@ -0,0 +1,14 @@
+{
+  "bots": [
+    {
+      "name": "allow-git-clients",
+      "action": "ALLOW",
+      "expression": {
+        "all": [
+          "userAgent.startsWith(\"git/\") || userAgent.contains(\"libgit\")",
+          "\"Git-Protocol\" in headers && headers[\"Git-Protocol\"] == \"version=2\""
+        ]
+      }
+    }
+  ]
+}

lib/config/testdata/good/git_client.yaml

@@ -0,0 +1,8 @@
+bots:
+- name: allow-git-clients
+  action: ALLOW
+  expression:
+    all:
+    - userAgent.startsWith("git/") || userAgent.contains("libgit")
+    - >
+      "Git-Protocol" in headers && headers["Git-Protocol"] == "version=2"

lib/config/testdata/good/import_filesystem.json

@@ -0,0 +1,7 @@
+{
+  "bots": [
+    {
+      "import": "./testdata/hack-test.json"
+    }
+  ]
+}

lib/config/testdata/good/import_filesystem.yaml

@@ -0,0 +1,2 @@
+bots:
+- import: ./testdata/hack-test.yaml

lib/config/testdata/good/import_keep_internet_working.json

@@ -0,0 +1,7 @@
+{
+  "bots": [
+    {
+      "import": "(data)/common/keep-internet-working.yaml"
+    }
+  ]
+}

lib/config/testdata/good/import_keep_internet_working.yaml

@@ -0,0 +1,2 @@
+bots:
+- import: (data)/common/keep-internet-working.yaml

lib/config/testdata/good/impressum.yaml

@@ -0,0 +1,10 @@
+bots:
+  - name: simple
+    action: CHALLENGE
+    user_agent_regex: Mozilla
+
+impressum:
+  footer: "Hi these are WORDS on the INTERNET."
+  page:
+    title: Test
+    body: <p>This is a test</p>

lib/config/testdata/good/logging-file.yaml

@@ -0,0 +1,15 @@
+bots:
+  - name: simple
+    action: CHALLENGE
+    user_agent_regex: Mozilla
+
+logs:
+  sink: "file"
+  parameters:
+    file: "/var/log/botstopper/default.log"
+    maxBackups: 3 # keep at least 3 old copies
+    maxBytes: 67108864 # each file can have up to 64 MB of logs
+    maxAge: 7 # rotate files out every n days
+    oldFileTimeFormat: 2006-01-02T15-04-05 # RFC 3339-ish
+    compress: true
+    useLocalTime: false # timezone for rotated files is UTC

lib/config/testdata/good/logging-stdio.yaml

@@ -0,0 +1,7 @@
+bots:
+  - name: simple
+    action: CHALLENGE
+    user_agent_regex: Mozilla
+
+logging:
+  sink: "stdio"

lib/config/testdata/good/no-thresholds.yaml

@@ -0,0 +1,8 @@
+bots:
+  - name: simple-weight-adjust
+    action: WEIGH
+    user_agent_regex: Mozilla
+    weight:
+      adjust: 5
+
+thresholds: []

lib/config/testdata/good/old_xesite.json

@@ -0,0 +1,79 @@
+{
+  "bots": [
+    {
+      "name": "amazonbot",
+      "user_agent_regex": "Amazonbot",
+      "action": "DENY"
+    },
+    {
+      "name": "googlebot",
+      "user_agent_regex": "\\+http\\:\\/\\/www\\.google\\.com/bot\\.html",
+      "action": "ALLOW"
+    },
+    {
+      "name": "bingbot",
+      "user_agent_regex": "\\+http\\:\\/\\/www\\.bing\\.com/bingbot\\.htm",
+      "action": "ALLOW"
+    },
+    {
+      "name": "qwantbot",
+      "user_agent_regex": "\\+https\\:\\/\\/help\\.qwant\\.com/bot/",
+      "action": "ALLOW"
+    },
+    {
+      "name": "discordbot",
+      "user_agent_regex": "Discordbot/2\\.0; \\+https\\:\\/\\/discordapp\\.com",
+      "action": "ALLOW"
+    },
+    {
+      "name": "blueskybot",
+      "user_agent_regex": "Bluesky Cardyb",
+      "action": "ALLOW"
+    },
+    {
+      "name": "us-artificial-intelligence-scraper",
+      "user_agent_regex": "\\+https\\:\\/\\/github\\.com\\/US-Artificial-Intelligence\\/scraper",
+      "action": "DENY"
+    },
+    {
+      "name": "well-known",
+      "path_regex": "^/.well-known/.*$",
+      "action": "ALLOW"
+    },
+    {
+      "name": "favicon",
+      "path_regex": "^/favicon.ico$",
+      "action": "ALLOW"
+    },
+    {
+      "name": "robots-txt",
+      "path_regex": "^/robots.txt$",
+      "action": "ALLOW"
+    },
+    {
+      "name": "rss-readers",
+      "path_regex": ".*\\.(rss|xml|atom|json)$",
+      "action": "ALLOW"
+    },
+    {
+      "name": "lightpanda",
+      "user_agent_regex": "^Lightpanda/.*$",
+      "action": "DENY"
+    },
+    {
+      "name": "headless-chrome",
+      "user_agent_regex": "HeadlessChrome",
+      "action": "DENY"
+    },
+    {
+      "name": "headless-chromium",
+      "user_agent_regex": "HeadlessChromium",
+      "action": "DENY"
+    },
+    {
+      "name": "generic-browser",
+      "user_agent_regex": "Mozilla",
+      "action": "CHALLENGE"
+    }
+  ]
+}

lib/config/testdata/good/opengraph_all_good.yaml

@@ -0,0 +1,12 @@
+bots:
+  - name: everything
+    user_agent_regex: .*
+    action: DENY
+
+openGraph:
+  enabled: true
+  considerHost: false
+  ttl: 1h
+  default:
+    "og:title": "Xe's magic land of fun"
+    "og:description": "We're no strangers to love, you know the rules and so do I"

lib/config/testdata/good/simple-weight.yaml

@@ -0,0 +1,6 @@
+bots:
+  - name: simple-weight-adjust
+    action: WEIGH
+    user_agent_regex: Mozilla
+    weight:
+      adjust: 5

lib/config/testdata/good/status-codes-paranoid.json

@@ -0,0 +1,13 @@
+{
+  "bots": [
+    {
+      "name": "everything",
+      "user_agent_regex": ".*",
+      "action": "DENY"
+    }
+  ],
+  "status_codes": {
+    "CHALLENGE": 200,
+    "DENY": 200
+  }
+}

lib/config/testdata/good/status-codes-paranoid.yaml

@@ -0,0 +1,8 @@
+bots:
+- name: everything
+  user_agent_regex: .*
+  action: DENY
+
+status_codes:
+  CHALLENGE: 200
+  DENY: 200

lib/config/testdata/good/status-codes-rfc.json

@@ -0,0 +1,13 @@
+{
+  "bots": [
+    {
+      "name": "everything",
+      "user_agent_regex": ".*",
+      "action": "DENY"
+    }
+  ],
+  "status_codes": {
+    "CHALLENGE": 403,
+    "DENY": 403
+  }
+}

lib/config/testdata/good/status-codes-rfc.yaml

@@ -0,0 +1,8 @@
+bots:
+- name: everything
+  user_agent_regex: .*
+  action: DENY
+
+status_codes:
+  CHALLENGE: 403
+  DENY: 403

lib/config/testdata/good/thresholds.yaml

@@ -0,0 +1,38 @@
+bots:
+  - name: simple-weight-adjust
+    action: WEIGH
+    user_agent_regex: Mozilla
+    weight:
+      adjust: 5
+
+thresholds:
+  - name: minimal-suspicion
+    expression: weight < 0
+    action: ALLOW
+  - name: mild-suspicion
+    expression:
+      all:
+        - weight >= 0
+        - weight < 10
+    action: CHALLENGE
+    challenge:
+      algorithm: metarefresh
+      difficulty: 1
+      report_as: 1
+  - name: moderate-suspicion
+    expression:
+      all:
+        - weight >= 10
+        - weight < 20
+    action: CHALLENGE
+    challenge:
+      algorithm: fast
+      difficulty: 2
+      report_as: 2
+  - name: extreme-suspicion
+    expression: weight >= 20
+    action: CHALLENGE
+    challenge:
+      algorithm: fast
+      difficulty: 4
+      report_as: 4

lib/config/testdata/good/weight-no-weight.yaml

@@ -0,0 +1,4 @@
+bots:
+  - name: weight
+    action: WEIGH
+    user_agent_regex: Mozilla