feat: writing logs to the filesystem with rotation support (#1299)
* refactor: move lib/policy/config to lib/config Signed-off-by: Xe Iaso <me@xeiaso.net> * refactor: don't set global loggers anymore Ref #864 You were right @kotx, it is a bad idea to set the global logger instance. Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(config): add log sink support Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: update spelling Signed-off-by: Xe Iaso <me@xeiaso.net> * chore(test): go mod tidy Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: update spelling Signed-off-by: Xe Iaso <me@xeiaso.net> * docs(admin/policies): add logging block documentation Signed-off-by: Xe Iaso <me@xeiaso.net> * docs: update CHANGELOG Signed-off-by: Xe Iaso <me@xeiaso.net> * fix(cmd/anubis): revert this change, it's meant to be its own PR Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: go mod tidy Signed-off-by: Xe Iaso <me@xeiaso.net> * test: add file logging smoke test Assisted-by: GLM 4.6 via Claude Code Signed-off-by: Xe Iaso <me@xeiaso.net> * fix: don't expose the old log file time format string Signed-off-by: Xe Iaso <me@xeiaso.net> --------- Signed-off-by: Xe Iaso <me@xeiaso.net>
This commit is contained in:
Xe Iaso
GitHub
parent
a709a2b2da
commit
f032d5d0ac
118 changed files with 789 additions and 65 deletions
|
|
@ -42,11 +42,13 @@ require (
|
|||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/djherbis/times v1.6.0 // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/ebitengine/purego v0.9.1 // indirect
|
||||
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fahedouch/go-logrotate v0.3.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/gaissmai/bart v0.26.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
|
|
|
|||
|
|
@ -82,6 +82,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
|
|||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
|
||||
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
|
||||
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
|
||||
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
|
||||
|
|
@ -98,6 +100,8 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt
|
|||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk=
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
|
||||
github.com/fahedouch/go-logrotate v0.3.0 h1:XP+dHIDgWZ1ckz43mG6gl5ASer3PZDVr755SVMyzaUQ=
|
||||
github.com/fahedouch/go-logrotate v0.3.0/go.mod h1:X49m0bvPLkk71MHNCQ1yEfVEw8W/u+qvHa/hOnhCYf4=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0=
|
||||
|
|
@ -251,6 +255,7 @@ golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
|
|||
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
|
|
@ -271,6 +276,8 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j
|
|||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
|
|||
18
test/log-file/anubis.yaml
Normal file
18
test/log-file/anubis.yaml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
bots:
|
||||
- name: challenge
|
||||
user_agent_regex: CHALLENGE
|
||||
action: CHALLENGE
|
||||
|
||||
status_codes:
|
||||
CHALLENGE: 200
|
||||
DENY: 403
|
||||
|
||||
logging:
|
||||
sink: file
|
||||
parameters:
|
||||
file: "./var/anubis.log"
|
||||
maxBackups: 3 # keep at least 3 old copies
|
||||
maxBytes: 67108864 # each file can have up to 64 Mi of logs
|
||||
maxAge: 7 # rotate files out every n days
|
||||
compress: true
|
||||
useLocalTime: false # timezone for rotated files is UTC
|
||||
178
test/log-file/input.txt
Normal file
178
test/log-file/input.txt
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
/wiki//bin
|
||||
/wiki//boot
|
||||
/wiki//dev
|
||||
/wiki//dev/de
|
||||
/wiki//dev/en
|
||||
/wiki//dev/en-ca
|
||||
/wiki//dev/es
|
||||
/wiki//dev/fr
|
||||
/wiki//dev/hr
|
||||
/wiki//dev/hu
|
||||
/wiki//dev/it
|
||||
/wiki//dev/ja
|
||||
/wiki//dev/ko
|
||||
/wiki//dev/pl
|
||||
/wiki//dev/pt-br
|
||||
/wiki//dev/ro
|
||||
/wiki//dev/ru
|
||||
/wiki//dev/sv
|
||||
/wiki//dev/uk
|
||||
/wiki//dev/zh-cn
|
||||
/wiki//etc
|
||||
/wiki//etc/conf.d
|
||||
/wiki//etc/env.d
|
||||
/wiki//etc/fstab
|
||||
/wiki//etc/fstab/de
|
||||
/wiki//etc/fstab/en
|
||||
/wiki//etc/fstab/es
|
||||
/wiki//etc/fstab/fr
|
||||
/wiki//etc/fstab/hu
|
||||
/wiki//etc/fstab/it
|
||||
/wiki//etc/fstab/ja
|
||||
/wiki//etc/fstab/ko
|
||||
/wiki//etc/fstab/ru
|
||||
/wiki//etc/fstab/sv
|
||||
/wiki//etc/fstab/uk
|
||||
/wiki//etc/fstab/zh-cn
|
||||
/wiki//etc/hosts
|
||||
/wiki//etc/local.d
|
||||
/wiki//etc/make.conf
|
||||
/wiki//etc/portage
|
||||
/wiki//etc/portage/bashrc
|
||||
/wiki//etc/portage/Bashrc
|
||||
/wiki//etc/portage/binrepos.conf
|
||||
/wiki//etc/portage/binrepos.conf/en
|
||||
/wiki//etc/portage/binrepos.conf/hu
|
||||
/wiki//etc/portage/binrepos.conf/ja
|
||||
/wiki//etc/portage/binrepos.conf/ru
|
||||
/wiki//etc/portage/categories
|
||||
/wiki//etc/portage/color.map
|
||||
/wiki//etc/portage/env
|
||||
/wiki//etc/portage/img/ico.png
|
||||
/wiki//etc/portage/license_groups
|
||||
/wiki//etc/portage/make.conf
|
||||
/wiki//etc/portage/make.conf/de
|
||||
/wiki//etc/portage/make.conf/de/etc/portage/make.conf
|
||||
/wiki//etc/portage/make.conf/en
|
||||
/wiki//etc/portage/make.conf/es
|
||||
/wiki//etc/portage/make.conf/fr
|
||||
/wiki//etc/portage/make.conf/hu
|
||||
/wiki//etc/portage/make.conf/it
|
||||
/wiki//etc/portage/make.conf/it/var/db/repos/gentoo/licenses
|
||||
/wiki//etc/portage/make.conf/ja
|
||||
/wiki//etc/portage/make.conf/pl
|
||||
/wiki//etc/portage/make.conf/ru
|
||||
/wiki//etc/portage/make.conf/uk
|
||||
/wiki//etc/portage/make.conf/zh-cn
|
||||
/wiki//etc/portage/make.profile
|
||||
/wiki//etc/portage/mirrors
|
||||
/wiki//etc/portage/modules
|
||||
/wiki//etc/portage/package.accept_keywords
|
||||
/wiki//etc/portage/package.env
|
||||
/wiki//etc/portage/package.license
|
||||
/wiki//etc/portage/package.license/en
|
||||
/wiki//etc/portage/package.license/es
|
||||
/wiki//etc/portage/package.license/hu
|
||||
/wiki//etc/portage/package.license/ja
|
||||
/wiki//etc/portage/package.mask
|
||||
/wiki//etc/portage/package.mask/en
|
||||
/wiki//etc/portage/package.mask/hu
|
||||
/wiki//etc/portage/package.mask/ja
|
||||
/wiki//etc/portage/package.properties
|
||||
/wiki//etc/portage/package.unmask
|
||||
/wiki//etc/portage/package.use
|
||||
/wiki//etc/portage/package.use/de
|
||||
/wiki//etc/portage/package.use/en
|
||||
/wiki//etc/portage/package.use/es
|
||||
/wiki//etc/portage/package.use/fr
|
||||
/wiki//etc/portage/package.use/hu
|
||||
/wiki//etc/portage/package.use/it
|
||||
/wiki//etc/portage/package.use/ja
|
||||
/wiki//etc/portage/package.use/ru
|
||||
/wiki//etc/portage/package.use/uk
|
||||
/wiki//etc/portage/package.use/zh-cn
|
||||
/wiki//etc/portage/patches
|
||||
/wiki//etc/portage/profile/make.defaults
|
||||
/wiki//etc/portage/profile/package.provided
|
||||
/wiki//etc/portage/profile/package.provided/etc/portage/profile/package.provided
|
||||
/wiki//etc/portage/profile/package.provided/etc/portage/profiles/package.provided
|
||||
/wiki//etc/portage/profile/package.use.mask
|
||||
/wiki//etc/portage/profiles/package.provided
|
||||
/wiki//etc/portage/profiles/package.use.mask
|
||||
/wiki//etc/portage/profiles/package.use.mask/etc/portage/profile/package.use.mask
|
||||
/wiki//etc/portage/profiles/package.use.mask/etc/portage/profiles/package.use.mask
|
||||
/wiki//etc/portage/profiles/use.mask
|
||||
/wiki//etc/portage/profile/use.mask
|
||||
/wiki//etc/portage/repos.conf
|
||||
/wiki//etc/portage/repos.conf/brother-overlay.conf
|
||||
/wiki//etc/portage/repos.conf/de
|
||||
/wiki//etc/portage/repos.conf/en
|
||||
/wiki//etc/portage/repos.conf/es
|
||||
/wiki//etc/portage/repos.conf/etc/portage/repos.conf/gentoo.conf
|
||||
/wiki//etc/portage/repos.conf/fr
|
||||
/wiki//etc/portage/repos.conf/fr/etc/portage/repos.conf/gentoo.conf
|
||||
/wiki//etc/portage/repos.conf/gentoo.conf
|
||||
/wiki//etc/portage/repos.conf/gentoo.conf/etc/portage/repos.conf/gentoo.conf
|
||||
/wiki//etc/portage/repos.conf/hr
|
||||
/wiki//etc/portage/repos.conf/hu
|
||||
/wiki//etc/portage/repos.conf/it
|
||||
/wiki//etc/portage/repos.conf/ja
|
||||
/wiki//etc/portage/repos.conf/ko
|
||||
/wiki//etc/portage/repos.conf/pl
|
||||
/wiki//etc/portage/repos.conf/pt-br
|
||||
/wiki//etc/portage/repos.conf/ru
|
||||
/wiki//etc/portage/repos.conf/uk
|
||||
/wiki//etc/portage/repos.conf/zh-cn
|
||||
/wiki//etc/portage/savedconfig
|
||||
/wiki//etc/portage/sets
|
||||
/wiki//etc/profile
|
||||
/wiki//etc/profile.env
|
||||
/wiki//etc/sandbox.conf
|
||||
/wiki//home
|
||||
/wiki//lib
|
||||
/wiki//lib64
|
||||
/wiki//media
|
||||
/wiki//mnt
|
||||
/wiki//opt
|
||||
/wiki//proc
|
||||
/wiki//proc/config.gz
|
||||
/wiki//run
|
||||
/wiki//sbin
|
||||
/wiki//srv
|
||||
/wiki//sys
|
||||
/wiki//tmp
|
||||
/wiki//usr
|
||||
/wiki//usr/bin
|
||||
/wiki//usr_move
|
||||
/wiki//usr/portage
|
||||
/wiki//usr/portage/distfiles
|
||||
/wiki//usr/portage/licenses
|
||||
/wiki//usr/portage/metadata
|
||||
/wiki//usr/portage/metadata/md5-cache
|
||||
/wiki//usr/portage/metadata/md5-cache/usr/portage/metadata/md5-cache
|
||||
/wiki//usr/portage/metadata/md5-cache/var/db/repos/gentoo//metadata/md5-cache
|
||||
/wiki//usr/portage/packages
|
||||
/wiki//usr/portage/profiles
|
||||
/wiki//usr/portage/profiles/license_groups
|
||||
/wiki//usr/portage/profiles/license_groups/usr/portage/profiles/license_groups
|
||||
/wiki//usr/portage/profiles/license_groups/var/db/repos/gentoo//profiles/license_groups
|
||||
/wiki//usr/share/doc/
|
||||
/wiki//var/cache/binpkgs
|
||||
/wiki//var/cache/distfiles
|
||||
/wiki//var/db/pkg
|
||||
/wiki//var/db/pkg%22
|
||||
/wiki//var/db/repos/gentoo
|
||||
/wiki//var/db/repos/gentoo/licenses
|
||||
/wiki//var/db/repos/gentoo/licenses/var/db/repos/gentoo//licenses
|
||||
/wiki//var/db/repos/gentoo/licenses/var/db/repos/gentoo/licenses
|
||||
/wiki//var/db/repos/gentoo/metadata
|
||||
/wiki//var/db/repos/gentoo/metadata/md5-cache
|
||||
/wiki//var/db/repos/gentoo/metadata/var/db/repos/gentoo//metadata
|
||||
/wiki//var/db/repos/gentoo/metadata/var/db/repos/gentoo/metadata
|
||||
/wiki//var/db/repos/gentoo/profiles
|
||||
/wiki//var/db/repos/gentoo/profiles/license_groups
|
||||
/wiki//var/db/repos/gentoo/profiles/package.mask
|
||||
/wiki//var/lib/portage
|
||||
/wiki//var/lib/portage/world
|
||||
/wiki//var/run
|
||||
/gcc-bugs/bug-122002-4@http.gcc.gnu.org%2Fbugzilla%2F/T/
|
||||
88
test/log-file/test.mjs
Normal file
88
test/log-file/test.mjs
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
import { statSync } from "fs";
|
||||
|
||||
async function getPage(path) {
|
||||
return fetch(`http://localhost:8923${path}`, {
|
||||
headers: {
|
||||
'User-Agent': 'CHALLENGE'
|
||||
}
|
||||
})
|
||||
.then(resp => {
|
||||
if (resp.status !== 200) {
|
||||
throw new Error(`wanted status 200, got status: ${resp.status}`);
|
||||
}
|
||||
return resp;
|
||||
})
|
||||
.then(resp => resp.text());
|
||||
}
|
||||
|
||||
async function getFileSize(filePath) {
|
||||
try {
|
||||
return statSync(filePath).size;
|
||||
} catch (error) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const logFilePath = "./var/anubis.log";
|
||||
|
||||
// Get initial log file size
|
||||
const initialSize = await getFileSize(logFilePath);
|
||||
console.log(`Initial log file size: ${initialSize} bytes`);
|
||||
|
||||
// Make 35 requests with different paths
|
||||
const requests = [];
|
||||
for (let i = 0; i < 35; i++) {
|
||||
requests.push(`/test${i}`);
|
||||
}
|
||||
|
||||
const resultSheet = {};
|
||||
let failed = false;
|
||||
|
||||
for (const path of requests) {
|
||||
try {
|
||||
const resp = await getPage(path);
|
||||
resultSheet[path] = {
|
||||
success: true,
|
||||
line: resp.split("\n")[0],
|
||||
};
|
||||
} catch (error) {
|
||||
resultSheet[path] = {
|
||||
success: false,
|
||||
error: error.message,
|
||||
};
|
||||
console.log(`✗ Request to ${path} failed: ${error.message}`);
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check final log file size
|
||||
const finalSize = await getFileSize(logFilePath);
|
||||
console.log(`Final log file size: ${finalSize} bytes`);
|
||||
console.log(`Size increase: ${finalSize - initialSize} bytes`);
|
||||
|
||||
// Verify that log file size increased
|
||||
if (finalSize <= initialSize) {
|
||||
console.error("ERROR: Log file size did not increase after making requests!");
|
||||
failed = true;
|
||||
}
|
||||
|
||||
let successCount = 0;
|
||||
for (let [k, v] of Object.entries(resultSheet)) {
|
||||
if (!v.success) {
|
||||
console.error({ path: k, error: v.error });
|
||||
} else {
|
||||
successCount++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Successful requests: ${successCount}/${requests.length}`);
|
||||
|
||||
if (failed) {
|
||||
console.error("Test failed: Some requests failed or log file size did not increase");
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log("Test passed: All requests succeeded and log file size increased");
|
||||
process.exit(0);
|
||||
}
|
||||
})();
|
||||
25
test/log-file/test.sh
Executable file
25
test/log-file/test.sh
Executable file
|
|
@ -0,0 +1,25 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
function cleanup() {
|
||||
pkill -P $$
|
||||
}
|
||||
|
||||
trap cleanup EXIT SIGINT
|
||||
|
||||
# Build static assets
|
||||
(cd ../.. && npm ci && npm run assets)
|
||||
|
||||
go tool anubis --help 2>/dev/null || :
|
||||
|
||||
go run ../cmd/httpdebug &
|
||||
|
||||
go tool anubis \
|
||||
--policy-fname ./anubis.yaml \
|
||||
--use-remote-address \
|
||||
--target=http://localhost:3923 &
|
||||
|
||||
sleep 2
|
||||
|
||||
backoff-retry node ./test.mjs
|
||||
2
test/log-file/var/.gitignore
vendored
Normal file
2
test/log-file/var/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
*
|
||||
!.gitignore
|
||||
Loading…
Add table
Add a link
Reference in a new issue