feat: writing logs to the filesystem with rotation support (#1299)
* refactor: move lib/policy/config to lib/config Signed-off-by: Xe Iaso <me@xeiaso.net> * refactor: don't set global loggers anymore Ref #864 You were right @kotx, it is a bad idea to set the global logger instance. Signed-off-by: Xe Iaso <me@xeiaso.net> * feat(config): add log sink support Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: update spelling Signed-off-by: Xe Iaso <me@xeiaso.net> * chore(test): go mod tidy Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: update spelling Signed-off-by: Xe Iaso <me@xeiaso.net> * docs(admin/policies): add logging block documentation Signed-off-by: Xe Iaso <me@xeiaso.net> * docs: update CHANGELOG Signed-off-by: Xe Iaso <me@xeiaso.net> * fix(cmd/anubis): revert this change, it's meant to be its own PR Signed-off-by: Xe Iaso <me@xeiaso.net> * chore: go mod tidy Signed-off-by: Xe Iaso <me@xeiaso.net> * test: add file logging smoke test Assisted-by: GLM 4.6 via Claude Code Signed-off-by: Xe Iaso <me@xeiaso.net> * fix: don't expose the old log file time format string Signed-off-by: Xe Iaso <me@xeiaso.net> --------- Signed-off-by: Xe Iaso <me@xeiaso.net>
This commit is contained in:
Xe Iaso
GitHub
parent
a709a2b2da
commit
f032d5d0ac
118 changed files with 789 additions and 65 deletions
18
test/log-file/anubis.yaml
Normal file
18
test/log-file/anubis.yaml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
bots:
|
||||
- name: challenge
|
||||
user_agent_regex: CHALLENGE
|
||||
action: CHALLENGE
|
||||
|
||||
status_codes:
|
||||
CHALLENGE: 200
|
||||
DENY: 403
|
||||
|
||||
logging:
|
||||
sink: file
|
||||
parameters:
|
||||
file: "./var/anubis.log"
|
||||
maxBackups: 3 # keep at least 3 old copies
|
||||
maxBytes: 67108864 # each file can have up to 64 Mi of logs
|
||||
maxAge: 7 # rotate files out every n days
|
||||
compress: true
|
||||
useLocalTime: false # timezone for rotated files is UTC
|
||||
178
test/log-file/input.txt
Normal file
178
test/log-file/input.txt
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
/wiki//bin
|
||||
/wiki//boot
|
||||
/wiki//dev
|
||||
/wiki//dev/de
|
||||
/wiki//dev/en
|
||||
/wiki//dev/en-ca
|
||||
/wiki//dev/es
|
||||
/wiki//dev/fr
|
||||
/wiki//dev/hr
|
||||
/wiki//dev/hu
|
||||
/wiki//dev/it
|
||||
/wiki//dev/ja
|
||||
/wiki//dev/ko
|
||||
/wiki//dev/pl
|
||||
/wiki//dev/pt-br
|
||||
/wiki//dev/ro
|
||||
/wiki//dev/ru
|
||||
/wiki//dev/sv
|
||||
/wiki//dev/uk
|
||||
/wiki//dev/zh-cn
|
||||
/wiki//etc
|
||||
/wiki//etc/conf.d
|
||||
/wiki//etc/env.d
|
||||
/wiki//etc/fstab
|
||||
/wiki//etc/fstab/de
|
||||
/wiki//etc/fstab/en
|
||||
/wiki//etc/fstab/es
|
||||
/wiki//etc/fstab/fr
|
||||
/wiki//etc/fstab/hu
|
||||
/wiki//etc/fstab/it
|
||||
/wiki//etc/fstab/ja
|
||||
/wiki//etc/fstab/ko
|
||||
/wiki//etc/fstab/ru
|
||||
/wiki//etc/fstab/sv
|
||||
/wiki//etc/fstab/uk
|
||||
/wiki//etc/fstab/zh-cn
|
||||
/wiki//etc/hosts
|
||||
/wiki//etc/local.d
|
||||
/wiki//etc/make.conf
|
||||
/wiki//etc/portage
|
||||
/wiki//etc/portage/bashrc
|
||||
/wiki//etc/portage/Bashrc
|
||||
/wiki//etc/portage/binrepos.conf
|
||||
/wiki//etc/portage/binrepos.conf/en
|
||||
/wiki//etc/portage/binrepos.conf/hu
|
||||
/wiki//etc/portage/binrepos.conf/ja
|
||||
/wiki//etc/portage/binrepos.conf/ru
|
||||
/wiki//etc/portage/categories
|
||||
/wiki//etc/portage/color.map
|
||||
/wiki//etc/portage/env
|
||||
/wiki//etc/portage/img/ico.png
|
||||
/wiki//etc/portage/license_groups
|
||||
/wiki//etc/portage/make.conf
|
||||
/wiki//etc/portage/make.conf/de
|
||||
/wiki//etc/portage/make.conf/de/etc/portage/make.conf
|
||||
/wiki//etc/portage/make.conf/en
|
||||
/wiki//etc/portage/make.conf/es
|
||||
/wiki//etc/portage/make.conf/fr
|
||||
/wiki//etc/portage/make.conf/hu
|
||||
/wiki//etc/portage/make.conf/it
|
||||
/wiki//etc/portage/make.conf/it/var/db/repos/gentoo/licenses
|
||||
/wiki//etc/portage/make.conf/ja
|
||||
/wiki//etc/portage/make.conf/pl
|
||||
/wiki//etc/portage/make.conf/ru
|
||||
/wiki//etc/portage/make.conf/uk
|
||||
/wiki//etc/portage/make.conf/zh-cn
|
||||
/wiki//etc/portage/make.profile
|
||||
/wiki//etc/portage/mirrors
|
||||
/wiki//etc/portage/modules
|
||||
/wiki//etc/portage/package.accept_keywords
|
||||
/wiki//etc/portage/package.env
|
||||
/wiki//etc/portage/package.license
|
||||
/wiki//etc/portage/package.license/en
|
||||
/wiki//etc/portage/package.license/es
|
||||
/wiki//etc/portage/package.license/hu
|
||||
/wiki//etc/portage/package.license/ja
|
||||
/wiki//etc/portage/package.mask
|
||||
/wiki//etc/portage/package.mask/en
|
||||
/wiki//etc/portage/package.mask/hu
|
||||
/wiki//etc/portage/package.mask/ja
|
||||
/wiki//etc/portage/package.properties
|
||||
/wiki//etc/portage/package.unmask
|
||||
/wiki//etc/portage/package.use
|
||||
/wiki//etc/portage/package.use/de
|
||||
/wiki//etc/portage/package.use/en
|
||||
/wiki//etc/portage/package.use/es
|
||||
/wiki//etc/portage/package.use/fr
|
||||
/wiki//etc/portage/package.use/hu
|
||||
/wiki//etc/portage/package.use/it
|
||||
/wiki//etc/portage/package.use/ja
|
||||
/wiki//etc/portage/package.use/ru
|
||||
/wiki//etc/portage/package.use/uk
|
||||
/wiki//etc/portage/package.use/zh-cn
|
||||
/wiki//etc/portage/patches
|
||||
/wiki//etc/portage/profile/make.defaults
|
||||
/wiki//etc/portage/profile/package.provided
|
||||
/wiki//etc/portage/profile/package.provided/etc/portage/profile/package.provided
|
||||
/wiki//etc/portage/profile/package.provided/etc/portage/profiles/package.provided
|
||||
/wiki//etc/portage/profile/package.use.mask
|
||||
/wiki//etc/portage/profiles/package.provided
|
||||
/wiki//etc/portage/profiles/package.use.mask
|
||||
/wiki//etc/portage/profiles/package.use.mask/etc/portage/profile/package.use.mask
|
||||
/wiki//etc/portage/profiles/package.use.mask/etc/portage/profiles/package.use.mask
|
||||
/wiki//etc/portage/profiles/use.mask
|
||||
/wiki//etc/portage/profile/use.mask
|
||||
/wiki//etc/portage/repos.conf
|
||||
/wiki//etc/portage/repos.conf/brother-overlay.conf
|
||||
/wiki//etc/portage/repos.conf/de
|
||||
/wiki//etc/portage/repos.conf/en
|
||||
/wiki//etc/portage/repos.conf/es
|
||||
/wiki//etc/portage/repos.conf/etc/portage/repos.conf/gentoo.conf
|
||||
/wiki//etc/portage/repos.conf/fr
|
||||
/wiki//etc/portage/repos.conf/fr/etc/portage/repos.conf/gentoo.conf
|
||||
/wiki//etc/portage/repos.conf/gentoo.conf
|
||||
/wiki//etc/portage/repos.conf/gentoo.conf/etc/portage/repos.conf/gentoo.conf
|
||||
/wiki//etc/portage/repos.conf/hr
|
||||
/wiki//etc/portage/repos.conf/hu
|
||||
/wiki//etc/portage/repos.conf/it
|
||||
/wiki//etc/portage/repos.conf/ja
|
||||
/wiki//etc/portage/repos.conf/ko
|
||||
/wiki//etc/portage/repos.conf/pl
|
||||
/wiki//etc/portage/repos.conf/pt-br
|
||||
/wiki//etc/portage/repos.conf/ru
|
||||
/wiki//etc/portage/repos.conf/uk
|
||||
/wiki//etc/portage/repos.conf/zh-cn
|
||||
/wiki//etc/portage/savedconfig
|
||||
/wiki//etc/portage/sets
|
||||
/wiki//etc/profile
|
||||
/wiki//etc/profile.env
|
||||
/wiki//etc/sandbox.conf
|
||||
/wiki//home
|
||||
/wiki//lib
|
||||
/wiki//lib64
|
||||
/wiki//media
|
||||
/wiki//mnt
|
||||
/wiki//opt
|
||||
/wiki//proc
|
||||
/wiki//proc/config.gz
|
||||
/wiki//run
|
||||
/wiki//sbin
|
||||
/wiki//srv
|
||||
/wiki//sys
|
||||
/wiki//tmp
|
||||
/wiki//usr
|
||||
/wiki//usr/bin
|
||||
/wiki//usr_move
|
||||
/wiki//usr/portage
|
||||
/wiki//usr/portage/distfiles
|
||||
/wiki//usr/portage/licenses
|
||||
/wiki//usr/portage/metadata
|
||||
/wiki//usr/portage/metadata/md5-cache
|
||||
/wiki//usr/portage/metadata/md5-cache/usr/portage/metadata/md5-cache
|
||||
/wiki//usr/portage/metadata/md5-cache/var/db/repos/gentoo//metadata/md5-cache
|
||||
/wiki//usr/portage/packages
|
||||
/wiki//usr/portage/profiles
|
||||
/wiki//usr/portage/profiles/license_groups
|
||||
/wiki//usr/portage/profiles/license_groups/usr/portage/profiles/license_groups
|
||||
/wiki//usr/portage/profiles/license_groups/var/db/repos/gentoo//profiles/license_groups
|
||||
/wiki//usr/share/doc/
|
||||
/wiki//var/cache/binpkgs
|
||||
/wiki//var/cache/distfiles
|
||||
/wiki//var/db/pkg
|
||||
/wiki//var/db/pkg%22
|
||||
/wiki//var/db/repos/gentoo
|
||||
/wiki//var/db/repos/gentoo/licenses
|
||||
/wiki//var/db/repos/gentoo/licenses/var/db/repos/gentoo//licenses
|
||||
/wiki//var/db/repos/gentoo/licenses/var/db/repos/gentoo/licenses
|
||||
/wiki//var/db/repos/gentoo/metadata
|
||||
/wiki//var/db/repos/gentoo/metadata/md5-cache
|
||||
/wiki//var/db/repos/gentoo/metadata/var/db/repos/gentoo//metadata
|
||||
/wiki//var/db/repos/gentoo/metadata/var/db/repos/gentoo/metadata
|
||||
/wiki//var/db/repos/gentoo/profiles
|
||||
/wiki//var/db/repos/gentoo/profiles/license_groups
|
||||
/wiki//var/db/repos/gentoo/profiles/package.mask
|
||||
/wiki//var/lib/portage
|
||||
/wiki//var/lib/portage/world
|
||||
/wiki//var/run
|
||||
/gcc-bugs/bug-122002-4@http.gcc.gnu.org%2Fbugzilla%2F/T/
|
||||
88
test/log-file/test.mjs
Normal file
88
test/log-file/test.mjs
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
import { statSync } from "fs";
|
||||
|
||||
async function getPage(path) {
|
||||
return fetch(`http://localhost:8923${path}`, {
|
||||
headers: {
|
||||
'User-Agent': 'CHALLENGE'
|
||||
}
|
||||
})
|
||||
.then(resp => {
|
||||
if (resp.status !== 200) {
|
||||
throw new Error(`wanted status 200, got status: ${resp.status}`);
|
||||
}
|
||||
return resp;
|
||||
})
|
||||
.then(resp => resp.text());
|
||||
}
|
||||
|
||||
async function getFileSize(filePath) {
|
||||
try {
|
||||
return statSync(filePath).size;
|
||||
} catch (error) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const logFilePath = "./var/anubis.log";
|
||||
|
||||
// Get initial log file size
|
||||
const initialSize = await getFileSize(logFilePath);
|
||||
console.log(`Initial log file size: ${initialSize} bytes`);
|
||||
|
||||
// Make 35 requests with different paths
|
||||
const requests = [];
|
||||
for (let i = 0; i < 35; i++) {
|
||||
requests.push(`/test${i}`);
|
||||
}
|
||||
|
||||
const resultSheet = {};
|
||||
let failed = false;
|
||||
|
||||
for (const path of requests) {
|
||||
try {
|
||||
const resp = await getPage(path);
|
||||
resultSheet[path] = {
|
||||
success: true,
|
||||
line: resp.split("\n")[0],
|
||||
};
|
||||
} catch (error) {
|
||||
resultSheet[path] = {
|
||||
success: false,
|
||||
error: error.message,
|
||||
};
|
||||
console.log(`✗ Request to ${path} failed: ${error.message}`);
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check final log file size
|
||||
const finalSize = await getFileSize(logFilePath);
|
||||
console.log(`Final log file size: ${finalSize} bytes`);
|
||||
console.log(`Size increase: ${finalSize - initialSize} bytes`);
|
||||
|
||||
// Verify that log file size increased
|
||||
if (finalSize <= initialSize) {
|
||||
console.error("ERROR: Log file size did not increase after making requests!");
|
||||
failed = true;
|
||||
}
|
||||
|
||||
let successCount = 0;
|
||||
for (let [k, v] of Object.entries(resultSheet)) {
|
||||
if (!v.success) {
|
||||
console.error({ path: k, error: v.error });
|
||||
} else {
|
||||
successCount++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Successful requests: ${successCount}/${requests.length}`);
|
||||
|
||||
if (failed) {
|
||||
console.error("Test failed: Some requests failed or log file size did not increase");
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log("Test passed: All requests succeeded and log file size increased");
|
||||
process.exit(0);
|
||||
}
|
||||
})();
|
||||
25
test/log-file/test.sh
Executable file
25
test/log-file/test.sh
Executable file
|
|
@ -0,0 +1,25 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
function cleanup() {
|
||||
pkill -P $$
|
||||
}
|
||||
|
||||
trap cleanup EXIT SIGINT
|
||||
|
||||
# Build static assets
|
||||
(cd ../.. && npm ci && npm run assets)
|
||||
|
||||
go tool anubis --help 2>/dev/null || :
|
||||
|
||||
go run ../cmd/httpdebug &
|
||||
|
||||
go tool anubis \
|
||||
--policy-fname ./anubis.yaml \
|
||||
--use-remote-address \
|
||||
--target=http://localhost:3923 &
|
||||
|
||||
sleep 2
|
||||
|
||||
backoff-retry node ./test.mjs
|
||||
2
test/log-file/var/.gitignore
vendored
Normal file
2
test/log-file/var/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
*
|
||||
!.gitignore
|
||||
Loading…
Add table
Add a link
Reference in a new issue