sophie/nuke
1
0
Fork 0
Code Packages 1 Activity Actions

feat(config): allow multi-level imports (#402)

Browse source 
* feat(config): allow multi-level imports

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(data): fix spelling of Marginalia

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>
This commit is contained in:
Xe Iaso 2025-05-02 13:57:20 -04:00 • committed by GitHub
parent f8e1000ab0
commit 6e82373718
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 89 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

43
data/botPolicies.json
View file

@ -1,40 +1,13 @@
{
"bots": [
{
"import": "(data)/bots/_deny-pathological.yaml"
},
{
"import": "(data)/bots/ai-robots-txt.yaml"
},
{
"import": "(data)/bots/cloudflare-workers.yaml"
},
{
"import": "(data)/bots/headless-browsers.yaml"
},
{
"import": "(data)/bots/us-ai-scraper.yaml"
},
{
"import": "(data)/crawlers/googlebot.yaml"
},
{
"import": "(data)/crawlers/bingbot.yaml"
},
{
"import": "(data)/crawlers/duckduckbot.yaml"
},
{
"import": "(data)/crawlers/qwantbot.yaml"
},
{
"import": "(data)/crawlers/internet-archive.yaml"
},
{
"import": "(data)/crawlers/kagibot.yaml"
},
{
"import": "(data)/crawlers/marginalia.yaml"
},
{
"import": "(data)/crawlers/mojeekbot.yaml"
"import": "(data)/crawlers/_allow-good.yaml"
},
{
"import": "(data)/common/keep-internet-working.yaml"
@ -45,5 +18,9 @@
"action": "CHALLENGE"
}
],
"dnsbl": false
}
"dnsbl": false,
"status_codes": {
"CHALLENGE": 200,
"DENY": 200
}
}

30
data/botPolicies.yaml
View file

@ -12,21 +12,23 @@
bots:
# Pathological bots to deny
- # This correlates to data/bots/ai-robots-txt.yaml in the source tree
import: (data)/bots/ai-robots-txt.yaml
- import: (data)/bots/cloudflare-workers.yaml
- import: (data)/bots/headless-browsers.yaml
- import: (data)/bots/us-ai-scraper.yaml
- # This correlates to data/bots/deny-pathological.yaml in the source tree
# https://github.com/TecharoHQ/anubis/blob/main/data/bots/deny-pathological.yaml
import: (data)/bots/_deny-pathological.yaml
# Search engines to allow
- import: (data)/crawlers/googlebot.yaml
- import: (data)/crawlers/bingbot.yaml
- import: (data)/crawlers/duckduckbot.yaml
- import: (data)/crawlers/qwantbot.yaml
- import: (data)/crawlers/internet-archive.yaml
- import: (data)/crawlers/kagibot.yaml
- import: (data)/crawlers/marginalia.yaml
- import: (data)/crawlers/mojeekbot.yaml
# Enforce https://github.com/ai-robots-txt/ai.robots.txt
- import: (data)/bots/ai-robots-txt.yaml
# Search engine crawlers to allow, defaults to:
# - Google (so they don't try to bypass Anubis)
# - Bing
# - DuckDuckGo
# - Qwant
# - The Internet Archive
# - Kagi
# - Marginalia
# - Mojeek
- import: (data)/crawlers/_allow-good.yaml
# Allow common "keeping the internet working" routes (well-known, favicon, robots.txt)
- import: (data)/common/keep-internet-working.yaml

3
data/bots/_deny-pathological.yaml Normal file
View file

@ -0,0 +1,3 @@
- import: (data)/bots/cloudflare-workers.yaml
- import: (data)/bots/headless-browsers.yaml
- import: (data)/bots/us-ai-scraper.yaml

8
data/crawlers/_allow-good.yaml Normal file
View file

@ -0,0 +1,8 @@
- import: (data)/crawlers/googlebot.yaml
- import: (data)/crawlers/bingbot.yaml
- import: (data)/crawlers/duckduckbot.yaml
- import: (data)/crawlers/qwantbot.yaml
- import: (data)/crawlers/internet-archive.yaml
- import: (data)/crawlers/kagibot.yaml
- import: (data)/crawlers/marginalia.yaml
- import: (data)/crawlers/mojeekbot.yaml

2
data/embed.go
View file

@ -3,6 +3,6 @@ package data
import "embed"
var (
//go:embed botPolicies.yaml botPolicies.json apps bots common crawlers
//go:embed botPolicies.yaml botPolicies.json all:apps all:bots all:common all:crawlers
BotPolicies embed.FS
)