From 359613f35ada5837012112ce77ff0cd5ece2f293 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Mon, 29 Dec 2025 12:10:17 -0500 Subject: [PATCH] feat: iplist2rule utility command (#1373) * feat: iplist2rule utility command Assisted-By: GLM 4.7 via Claude Code Signed-off-by: Xe Iaso * docs: update CHANGELOG Signed-off-by: Xe Iaso * chore: fix spelling Signed-off-by: Xe Iaso * chore: fix spelling again Signed-off-by: Xe Iaso * feat(iplist2rule): add comment describing how rule was generated Signed-off-by: Xe Iaso * docs: add iplist2rule docs Signed-off-by: Xe Iaso * chore: fix spelling Signed-off-by: Xe Iaso --------- Signed-off-by: Xe Iaso --- .github/actions/spelling/allow.txt | 5 ++ docs/docs/CHANGELOG.md | 1 + docs/docs/admin/iplist2rule.mdx | 50 ++++++++++++++ docs/docs/admin/robots2policy.mdx | 11 +-- utils/cmd/iplist2rule/blocklist.go | 57 ++++++++++++++++ utils/cmd/iplist2rule/main.go | 103 +++++++++++++++++++++++++++++ 6 files changed, 223 insertions(+), 4 deletions(-) create mode 100644 docs/docs/admin/iplist2rule.mdx create mode 100644 utils/cmd/iplist2rule/blocklist.go create mode 100644 utils/cmd/iplist2rule/main.go diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 5e3002b..9d7a56c 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -18,3 +18,8 @@ clampip pseudoprofound reimagining iocaine +admins +fout +iplist +NArg +blocklists diff --git a/docs/docs/CHANGELOG.md b/docs/docs/CHANGELOG.md index 0c4271c..2fba492 100644 --- a/docs/docs/CHANGELOG.md +++ b/docs/docs/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Add iplist2rule tool that lets admins turn an IP address blocklist into an Anubis ruleset. - Add Polish locale ([#1292](https://github.com/TecharoHQ/anubis/pull/1309)) diff --git a/docs/docs/admin/iplist2rule.mdx b/docs/docs/admin/iplist2rule.mdx new file mode 100644 index 0000000..4ff1d3d --- /dev/null +++ b/docs/docs/admin/iplist2rule.mdx @@ -0,0 +1,50 @@ +--- +title: iplist2rule CLI tool +--- + +The `iplist2rule` tool converts IP blocklists into Anubis challenge policies. It reads common IP block list formats and generates the appropriate Anubis policy file for IP address filtering. + +## Installation + +Install directly with Go + +```bash +go install github.com/TecharoHQ/anubis/utils/cmd/iplist2rule@latest +``` + +## Usage + +Basic conversion from URL: + +```bash +iplist2rule https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml +``` + +Explicitly allow every IP address on a list: + +```bash +iplist2rule --action ALLOW https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml +``` + +Add weight to requests matching IP addresses on a list: + +```bash +iplist2rule --action WEIGH --weight 20 https://raw.githubusercontent.com/7c/torfilter/refs/heads/main/lists/txt/torfilter-1m-flat.txt filter-tor.yaml +``` + +## Options + +| Flag | Description | Default | +| :------------ | :----------------------------------------------------------------------------------------------- | :-------------------------------- | +| `--action` | The Anubis action to take for the IP address in question, must be in ALL CAPS. | `DENY` (forbids traffic) | +| `--rule-name` | The name for the generated Anubis rule, should be in kebab-case. | (not set, inferred from filename) | +| `--weight` | When `--action=WEIGH`, how many weight points should be added or removed from matching requests? | 0 (not set) | + +## Using the Generated Policy + +Save the output and import it in your main policy file: + +```yaml +bots: + - import: "./filter-tor.yaml" +``` diff --git a/docs/docs/admin/robots2policy.mdx b/docs/docs/admin/robots2policy.mdx index 30f0eab..fdbab6f 100644 --- a/docs/docs/admin/robots2policy.mdx +++ b/docs/docs/admin/robots2policy.mdx @@ -12,6 +12,7 @@ Install directly with Go: ```bash go install github.com/TecharoHQ/anubis/cmd/robots2policy@latest ``` + ## Usage Basic conversion from URL: @@ -35,8 +36,8 @@ robots2policy -input robots.txt -action DENY -format json ## Options | Flag | Description | Default | -|-----------------------|--------------------------------------------------------------------|---------------------| -| `-input` | robots.txt file path or URL (use `-` for stdin) | *required* | +| --------------------- | ------------------------------------------------------------------ | ------------------- | +| `-input` | robots.txt file path or URL (use `-` for stdin) | _required_ | | `-output` | Output file (use `-` for stdout) | stdout | | `-format` | Output format: `yaml` or `json` | `yaml` | | `-action` | Action for disallowed paths: `ALLOW`, `DENY`, `CHALLENGE`, `WEIGH` | `CHALLENGE` | @@ -47,6 +48,7 @@ robots2policy -input robots.txt -action DENY -format json ## Example Input robots.txt: + ```txt User-agent: * Disallow: /admin/ @@ -57,6 +59,7 @@ Disallow: / ``` Generated policy: + ```yaml - name: robots-txt-policy-disallow-1 action: CHALLENGE @@ -77,8 +80,8 @@ Generated policy: Save the output and import it in your main policy file: ```yaml -import: - - path: "./robots-policy.yaml" +bots: + - import: "./robots-policy.yaml" ``` The tool handles wildcard patterns, user-agent specific rules, and blacklisted bots automatically. diff --git a/utils/cmd/iplist2rule/blocklist.go b/utils/cmd/iplist2rule/blocklist.go new file mode 100644 index 0000000..72bb47d --- /dev/null +++ b/utils/cmd/iplist2rule/blocklist.go @@ -0,0 +1,57 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "net/http" + "net/netip" + "strings" +) + +// FetchBlocklist reads the blocklist over HTTP and returns every non-commented +// line parsed as an IP address in CIDR notation. IPv4 addresses are returned as +// /32, IPv6 addresses as /128. +// +// This function was generated with GLM 4.7. +func FetchBlocklist(url string) ([]string, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP request failed with status: %s", resp.Status) + } + + var lines []string + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + // Skip empty lines and comments (lines starting with #) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + addr, err := netip.ParseAddr(line) + if err != nil { + // Skip lines that aren't valid IP addresses + continue + } + + var cidr string + if addr.Is4() { + cidr = fmt.Sprintf("%s/32", addr.String()) + } else { + cidr = fmt.Sprintf("%s/128", addr.String()) + } + lines = append(lines, cidr) + } + + if err := scanner.Err(); err != nil && err != io.EOF { + return nil, err + } + + return lines, nil +} diff --git a/utils/cmd/iplist2rule/main.go b/utils/cmd/iplist2rule/main.go new file mode 100644 index 0000000..db49d1a --- /dev/null +++ b/utils/cmd/iplist2rule/main.go @@ -0,0 +1,103 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "time" + + "github.com/TecharoHQ/anubis/lib/config" + "github.com/facebookgo/flagenv" + "sigs.k8s.io/yaml" +) + +type Rule struct { + Name string `yaml:"name" json:"name"` + Action config.Rule `yaml:"action" json:"action"` + RemoteAddr []string `json:"remote_addresses,omitempty" yaml:"remote_addresses,omitempty"` + Weight *config.Weight `json:"weight,omitempty" yaml:"weight,omitempty"` +} + +func init() { + flag.Usage = func() { + fmt.Printf(`Usage of %[1]s: + + %[1]s [flags] + +Grabs the contents of the blocklist, converts it to an Anubis ruleset, and writes it to filename. + +Flags: +`, filepath.Base(os.Args[0])) + + flag.PrintDefaults() + } +} + +var ( + action = flag.String("action", "DENY", "Anubis action to take (ALLOW / DENY / WEIGH)") + manualRuleName = flag.String("rule-name", "", "If set, prefer this name over inferring from filename") + weight = flag.Int("weight", 0, "If set to any number, add/subtract this many weight points when --action=WEIGH") +) + +func main() { + flagenv.Parse() + flag.Parse() + + if flag.NArg() != 2 { + flag.Usage() + os.Exit(2) + } + + blocklistURL := flag.Arg(0) + foutName := flag.Arg(1) + ruleName := strings.TrimSuffix(foutName, filepath.Ext(foutName)) + + if *manualRuleName != "" { + ruleName = *manualRuleName + } + + ruleAction := config.Rule(*action) + if err := ruleAction.Valid(); err != nil { + log.Fatalf("--action=%q is invalid: %v", *action, err) + } + + result := &Rule{ + Name: ruleName, + Action: ruleAction, + } + + if *weight != 0 { + if ruleAction != config.RuleWeigh { + log.Fatalf("used --weight=%d but --action=%s", *weight, *action) + } + + result.Weight = &config.Weight{ + Adjust: *weight, + } + } + + ips, err := FetchBlocklist(blocklistURL) + if err != nil { + log.Fatalf("can't fetch blocklist %s: %v", blocklistURL, err) + } + + result.RemoteAddr = ips + + fout, err := os.Create(foutName) + if err != nil { + log.Fatalf("can't create output file %q: %v", foutName, err) + } + defer fout.Close() + + fmt.Fprintf(fout, "# Generated by %s on %s from %s\n\n", filepath.Base(os.Args[0]), time.Now().Format(time.RFC3339), blocklistURL) + + data, err := yaml.Marshal([]*Rule{result}) + if err != nil { + log.Fatalf("can't marshal yaml") + } + + fout.Write(data) +}