sophie/nuke
1
0
Fork 0
Code Packages 1 Activity Actions

fix: SERVE_ROBOTS_TXT works again (#1229)

Browse source 
Signed-off-by: Xe Iaso <me@xeiaso.net>
This commit is contained in:
Xe Iaso 2025-10-31 09:08:33 -04:00 • committed by GitHub
parent 62c1b80189
commit 59f1e36167
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 73 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

8
test/robots_txt/anubis.yaml Normal file
View file

@ -0,0 +1,8 @@
bots:
- name: challenge
user_agent_regex: CHALLENGE
action: CHALLENGE
status_codes:
CHALLENGE: 200
DENY: 403

27
test/robots_txt/test.mjs Normal file
View file

@ -0,0 +1,27 @@
async function getRobotsTxt() {
return fetch("http://localhost:8923/robots.txt", {
headers: {
"Accept-Language": "en",
"User-Agent": "Mozilla/5.0",
}
})
.then(resp => {
if (resp.status !== 200) {
throw new Error(`wanted status 200, got status: ${resp.status}`);
}
return resp;
})
.then(resp => resp.text());
}
(async () => {
const page = await getRobotsTxt();
if (page.includes(`<html>`)) {
console.log(page)
throw new Error("serve robots.txt smoke test failed");
}
console.log("serve-robots-txt serves robots.txt");
process.exit(0);
})();

24
test/robots_txt/test.sh Executable file
View file

@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -euo pipefail
function cleanup() {
pkill -P $$
}
trap cleanup EXIT SIGINT
# Build static assets
(cd ../.. && npm ci && npm run assets)
go tool anubis --help 2>/dev/null || :
go run ../cmd/unixhttpd &
go tool anubis \
--policy-fname ./anubis.yaml \
--use-remote-address \
--serve-robots-txt \
--target=unix://$(pwd)/unixhttpd.sock &
backoff-retry node ./test.mjs

2
test/robots_txt/var/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
*
!.gitignore