sophie/nuke
1
0
Fork 0
Code Packages 1 Activity Actions

fix: SERVE_ROBOTS_TXT works again (#1229)

Browse source 
Signed-off-by: Xe Iaso <me@xeiaso.net>
This commit is contained in:
Xe Iaso 2025-10-31 09:08:33 -04:00 • committed by GitHub
parent 62c1b80189
commit 59f1e36167
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 73 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/smoke-tests.yml vendored
View file

@ -23,6 +23,7 @@ jobs:
- i18n
- palemoon/amd64
#- palemoon/i386
- robots_txt
runs-on: ubuntu-latest
steps:
- name: Checkout code

2
docs/docs/CHANGELOG.md
View file

@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
<!-- This changes the project to: -->
- Fix `SERVE_ROBOTS_TXT` setting file after the double slash fix broke it.
## v1.23.0: Lyse Hext
- Add default tencent cloud DENY rule.

9
lib/http.go
View file

@ -345,6 +345,15 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
// Forward robots.txt requests to mux when ServeRobotsTXT is enabled
if s.opts.ServeRobotsTXT {
path := strings.TrimPrefix(r.URL.Path, anubis.BasePrefix)
if path == "/robots.txt" || path == "/.well-known/robots.txt" {
s.mux.ServeHTTP(w, r)
return
}
}
s.maybeReverseProxyOrPage(w, r)
}

8
test/robots_txt/anubis.yaml Normal file
View file

@ -0,0 +1,8 @@
bots:
- name: challenge
user_agent_regex: CHALLENGE
action: CHALLENGE
status_codes:
CHALLENGE: 200
DENY: 403

27
test/robots_txt/test.mjs Normal file
View file

@ -0,0 +1,27 @@
async function getRobotsTxt() {
return fetch("http://localhost:8923/robots.txt", {
headers: {
"Accept-Language": "en",
"User-Agent": "Mozilla/5.0",
}
})
.then(resp => {
if (resp.status !== 200) {
throw new Error(`wanted status 200, got status: ${resp.status}`);
}
return resp;
})
.then(resp => resp.text());
}
(async () => {
const page = await getRobotsTxt();
if (page.includes(`<html>`)) {
console.log(page)
throw new Error("serve robots.txt smoke test failed");
}
console.log("serve-robots-txt serves robots.txt");
process.exit(0);
})();

24
test/robots_txt/test.sh Executable file
View file

@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -euo pipefail
function cleanup() {
pkill -P $$
}
trap cleanup EXIT SIGINT
# Build static assets
(cd ../.. && npm ci && npm run assets)
go tool anubis --help 2>/dev/null || :
go run ../cmd/unixhttpd &
go tool anubis \
--policy-fname ./anubis.yaml \
--use-remote-address \
--serve-robots-txt \
--target=unix://$(pwd)/unixhttpd.sock &
backoff-retry node ./test.mjs

2
test/robots_txt/var/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
*
!.gitignore