sophie/nuke
1
0
Fork 0
Code Packages 1 Activity Actions

fix(robots2policy): handle multiple user agents under one block (#925)

Browse source
This commit is contained in:
Jason Cameron 2025-09-06 22:35:19 -04:00 • committed by GitHub
parent 87c2f1e0e6
commit 82099d9e05
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 208 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

24
cmd/robots2policy/testdata/complex.yaml vendored
View file

@ -20,8 +20,8 @@
- action: CHALLENGE
expression:
all:
- userAgent.contains("Googlebot")
- path.startsWith("/search/")
- userAgent.contains("Googlebot")
- path.startsWith("/search/")
name: robots-txt-policy-disallow-6
- action: WEIGH
expression: userAgent.contains("Bingbot")
@ -31,14 +31,14 @@
- action: CHALLENGE
expression:
all:
- userAgent.contains("Bingbot")
- path.startsWith("/search/")
- userAgent.contains("Bingbot")
- path.startsWith("/search/")
name: robots-txt-policy-disallow-8
- action: CHALLENGE
expression:
all:
- userAgent.contains("Bingbot")
- path.startsWith("/admin/")
- userAgent.contains("Bingbot")
- path.startsWith("/admin/")
name: robots-txt-policy-disallow-9
- action: DENY
expression: userAgent.contains("BadBot")
@ -54,18 +54,18 @@
- action: CHALLENGE
expression:
all:
- userAgent.contains("TestBot")
- path.matches("^/.*/admin")
- userAgent.contains("TestBot")
- path.matches("^/.*/admin")
name: robots-txt-policy-disallow-13
- action: CHALLENGE
expression:
all:
- userAgent.contains("TestBot")
- path.matches("^/temp.*\\.html")
- userAgent.contains("TestBot")
- path.matches("^/temp.*\\.html")
name: robots-txt-policy-disallow-14
- action: CHALLENGE
expression:
all:
- userAgent.contains("TestBot")
- path.matches("^/file.\\.log")
- userAgent.contains("TestBot")
- path.matches("^/file.\\.log")
name: robots-txt-policy-disallow-15