feat(config): allow multi-level imports (#402)

* feat(config): allow multi-level imports

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(data): fix spelling of Marginalia

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>

data/botPolicies.yaml

@@ -12,21 +12,23 @@
 
 bots:
 # Pathological bots to deny
-- # This correlates to data/bots/ai-robots-txt.yaml in the source tree
-  import: (data)/bots/ai-robots-txt.yaml
-- import: (data)/bots/cloudflare-workers.yaml 
-- import: (data)/bots/headless-browsers.yaml
-- import: (data)/bots/us-ai-scraper.yaml
+- # This correlates to data/bots/deny-pathological.yaml in the source tree
+  # https://github.com/TecharoHQ/anubis/blob/main/data/bots/deny-pathological.yaml
+  import: (data)/bots/_deny-pathological.yaml
 
-# Search engines to allow
-- import: (data)/crawlers/googlebot.yaml
-- import: (data)/crawlers/bingbot.yaml
-- import: (data)/crawlers/duckduckbot.yaml
-- import: (data)/crawlers/qwantbot.yaml
-- import: (data)/crawlers/internet-archive.yaml
-- import: (data)/crawlers/kagibot.yaml
-- import: (data)/crawlers/marginalia.yaml
-- import: (data)/crawlers/mojeekbot.yaml
+# Enforce https://github.com/ai-robots-txt/ai.robots.txt
+- import: (data)/bots/ai-robots-txt.yaml
+
+# Search engine crawlers to allow, defaults to:
+#   - Google (so they don't try to bypass Anubis)
+#   - Bing
+#   - DuckDuckGo
+#   - Qwant
+#   - The Internet Archive
+#   - Kagi
+#   - Marginalia
+#   - Mojeek
+- import: (data)/crawlers/_allow-good.yaml
 
 # Allow common "keeping the internet working" routes (well-known, favicon, robots.txt)
 - import: (data)/common/keep-internet-working.yaml