This commit is contained in:
parent
d2205b11a7
commit
02b9aebbe5
341 changed files with 1571 additions and 32574 deletions
2
cmd/anubis/.gitignore
vendored
2
cmd/anubis/.gitignore
vendored
|
|
@ -1,2 +0,0 @@
|
|||
*.rpm
|
||||
anubis
|
||||
|
|
@ -10,14 +10,14 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/TecharoHQ/anubis/internal"
|
||||
"git.sad.ovh/sophie/nuke/internal"
|
||||
"github.com/facebookgo/flagenv"
|
||||
)
|
||||
|
||||
var (
|
||||
dockerAnnotations = flag.String("docker-annotations", os.Getenv("DOCKER_METADATA_OUTPUT_ANNOTATIONS"), "Docker image annotations")
|
||||
dockerLabels = flag.String("docker-labels", os.Getenv("DOCKER_METADATA_OUTPUT_LABELS"), "Docker image labels")
|
||||
dockerRepo = flag.String("docker-repo", "registry.int.xeserv.us/techaro/anubis", "Docker image repository for Anubis")
|
||||
dockerRepo = flag.String("docker-repo", "git.sad.ovh/sophie/nuke", "Docker image repository for Nuke")
|
||||
dockerTags = flag.String("docker-tags", os.Getenv("DOCKER_METADATA_OUTPUT_TAGS"), "newline separated docker tags including the registry name")
|
||||
githubEventName = flag.String("github-event-name", "", "GitHub event name")
|
||||
pullRequestID = flag.Int("pull-request-id", -1, "GitHub pull request ID")
|
||||
|
|
@ -33,9 +33,9 @@ func main() {
|
|||
koDockerRepo := strings.TrimSuffix(*dockerRepo, "/"+filepath.Base(*dockerRepo))
|
||||
|
||||
if *githubEventName == "pull_request" && *pullRequestID != -1 {
|
||||
*dockerRepo = fmt.Sprintf("ttl.sh/techaro/pr-%d/anubis", *pullRequestID)
|
||||
*dockerTags = fmt.Sprintf("ttl.sh/techaro/pr-%d/anubis:24h", *pullRequestID)
|
||||
koDockerRepo = fmt.Sprintf("ttl.sh/techaro/pr-%d", *pullRequestID)
|
||||
*dockerRepo = fmt.Sprintf("ttl.sh/sophie/pr-%d/nuke", *pullRequestID)
|
||||
*dockerTags = fmt.Sprintf("ttl.sh/sophie/pr-%d/nuke:24h", *pullRequestID)
|
||||
koDockerRepo = fmt.Sprintf("ttl.sh/sophie/pr-%d", *pullRequestID)
|
||||
|
||||
slog.Info(
|
||||
"Building image for pull request",
|
||||
|
|
@ -102,7 +102,7 @@ func main() {
|
|||
tags = append(tags, img.tag)
|
||||
}
|
||||
|
||||
output, err := run(fmt.Sprintf("ko build --platform=all --base-import-paths --tags=%q --image-user=1000 --image-annotation=%q --image-label=%q ./cmd/anubis | tail -n1", strings.Join(tags, ","), *dockerAnnotations, *dockerLabels))
|
||||
output, err := run(fmt.Sprintf("ko build --platform=all --base-import-paths --tags=%q --image-user=1000 --image-annotation=%q --image-label=%q ./cmd/nuke | tail -n1", strings.Join(tags, ","), *dockerAnnotations, *dockerLabels))
|
||||
if err != nil {
|
||||
log.Fatalf("can't run ko build, check stderr: %v", err)
|
||||
}
|
||||
|
|
@ -125,8 +125,8 @@ func parseImageList(imageList string) ([]image, error) {
|
|||
continue
|
||||
}
|
||||
|
||||
// reg.xeiaso.net/techaro/anubis:latest
|
||||
// repository: reg.xeiaso.net/techaro/anubis
|
||||
// reg.xeiaso.net/techaro/nuke:latest
|
||||
// repository: reg.xeiaso.net/techaro/nuke
|
||||
// tag: latest
|
||||
index := strings.LastIndex(img, ":")
|
||||
result = append(result, image{
|
||||
|
|
|
|||
2
cmd/nuke/.gitignore
vendored
Normal file
2
cmd/nuke/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
*.rpm
|
||||
nuke
|
||||
|
|
@ -27,14 +27,13 @@ import (
|
|||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/TecharoHQ/anubis"
|
||||
"github.com/TecharoHQ/anubis/data"
|
||||
"github.com/TecharoHQ/anubis/internal"
|
||||
libanubis "github.com/TecharoHQ/anubis/lib"
|
||||
"github.com/TecharoHQ/anubis/lib/config"
|
||||
botPolicy "github.com/TecharoHQ/anubis/lib/policy"
|
||||
"github.com/TecharoHQ/anubis/lib/thoth"
|
||||
"github.com/TecharoHQ/anubis/web"
|
||||
"git.sad.ovh/sophie/nuke"
|
||||
"git.sad.ovh/sophie/nuke/data"
|
||||
"git.sad.ovh/sophie/nuke/internal"
|
||||
libnuke "git.sad.ovh/sophie/nuke/lib"
|
||||
"git.sad.ovh/sophie/nuke/lib/config"
|
||||
botPolicy "git.sad.ovh/sophie/nuke/lib/policy"
|
||||
"git.sad.ovh/sophie/nuke/web"
|
||||
"github.com/facebookgo/flagenv"
|
||||
_ "github.com/joho/godotenv/autoload"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
|
@ -45,26 +44,26 @@ var (
|
|||
basePrefix = flag.String("base-prefix", "", "base prefix (root URL) the application is served under e.g. /myapp")
|
||||
bind = flag.String("bind", ":8923", "network address to bind HTTP to")
|
||||
bindNetwork = flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp")
|
||||
challengeDifficulty = flag.Int("difficulty", anubis.DefaultDifficulty, "difficulty of the challenge")
|
||||
cookieDomain = flag.String("cookie-domain", "", "if set, the top-level domain that the Anubis cookie will be valid for")
|
||||
challengeDifficulty = flag.Int("difficulty", nuke.DefaultDifficulty, "difficulty of the challenge")
|
||||
cookieDomain = flag.String("cookie-domain", "", "if set, the top-level domain that the Nuke cookie will be valid for")
|
||||
cookieDynamicDomain = flag.Bool("cookie-dynamic-domain", false, "if set, automatically set the cookie Domain value based on the request domain")
|
||||
cookieExpiration = flag.Duration("cookie-expiration-time", anubis.CookieDefaultExpirationTime, "The amount of time the authorization cookie is valid for")
|
||||
cookiePrefix = flag.String("cookie-prefix", anubis.CookieName, "prefix for browser cookies created by Anubis")
|
||||
cookiePartitioned = flag.Bool("cookie-partitioned", false, "if true, sets the partitioned flag on Anubis cookies, enabling CHIPS support")
|
||||
cookieExpiration = flag.Duration("cookie-expiration-time", nuke.CookieDefaultExpirationTime, "The amount of time the authorization cookie is valid for")
|
||||
cookiePrefix = flag.String("cookie-prefix", nuke.CookieName, "prefix for browser cookies created by Nuke")
|
||||
cookiePartitioned = flag.Bool("cookie-partitioned", false, "if true, sets the partitioned flag on Nuke cookies, enabling CHIPS support")
|
||||
difficultyInJWT = flag.Bool("difficulty-in-jwt", false, "if true, adds a difficulty field in the JWT claims")
|
||||
useSimplifiedExplanation = flag.Bool("use-simplified-explanation", false, "if true, replaces the text when clicking \"Why am I seeing this?\" with a more simplified text for a non-tech-savvy audience.")
|
||||
forcedLanguage = flag.String("forced-language", "", "if set, this language is being used instead of the one from the request's Accept-Language header")
|
||||
hs512Secret = flag.String("hs512-secret", "", "secret used to sign JWTs, uses ed25519 if not set")
|
||||
cookieSecure = flag.Bool("cookie-secure", true, "if true, sets the secure flag on Anubis cookies")
|
||||
cookieSameSite = flag.String("cookie-same-site", "None", "sets the same site option on Anubis cookies, will auto-downgrade None to Lax if cookie-secure is false. Valid values are None, Lax, Strict, and Default.")
|
||||
cookieSecure = flag.Bool("cookie-secure", true, "if true, sets the secure flag on Nuke cookies")
|
||||
cookieSameSite = flag.String("cookie-same-site", "None", "sets the same site option on Nuke cookies, will auto-downgrade None to Lax if cookie-secure is false. Valid values are None, Lax, Strict, and Default.")
|
||||
ed25519PrivateKeyHex = flag.String("ed25519-private-key-hex", "", "private key used to sign JWTs, if not set a random one will be assigned")
|
||||
ed25519PrivateKeyHexFile = flag.String("ed25519-private-key-hex-file", "", "file name containing value for ed25519-private-key-hex")
|
||||
metricsBind = flag.String("metrics-bind", ":9090", "network address to bind metrics to")
|
||||
metricsBindNetwork = flag.String("metrics-bind-network", "tcp", "network family for the metrics server to bind to")
|
||||
socketMode = flag.String("socket-mode", "0770", "socket mode (permissions) for unix domain sockets.")
|
||||
robotsTxt = flag.Bool("serve-robots-txt", false, "serve a robots.txt file that disallows all robots")
|
||||
policyFname = flag.String("policy-fname", "", "full path to anubis policy document (defaults to a sensible built-in policy)")
|
||||
redirectDomains = flag.String("redirect-domains", "", "list of domains separated by commas which anubis is allowed to redirect to. Leaving this unset allows any domain.")
|
||||
policyFname = flag.String("policy-fname", "", "full path to nuke policy document (defaults to a sensible built-in policy)")
|
||||
redirectDomains = flag.String("redirect-domains", "", "list of domains separated by commas which nuke is allowed to redirect to. Leaving this unset allows any domain.")
|
||||
slogLevel = flag.String("slog-level", "INFO", "logging level (see https://pkg.go.dev/log/slog#hdr-Levels)")
|
||||
stripBasePrefix = flag.Bool("strip-base-prefix", false, "if true, strips the base prefix from requests forwarded to the target server")
|
||||
target = flag.String("target", "http://localhost:3923", "target to reverse proxy to, set to an empty string to disable proxying when only using auth request")
|
||||
|
|
@ -72,22 +71,19 @@ var (
|
|||
targetHost = flag.String("target-host", "", "if set, the value of the Host header when forwarding requests to the target")
|
||||
targetInsecureSkipVerify = flag.Bool("target-insecure-skip-verify", false, "if true, skips TLS validation for the backend")
|
||||
targetDisableKeepAlive = flag.Bool("target-disable-keepalive", false, "if true, disables HTTP keep-alive for the backend")
|
||||
healthcheck = flag.Bool("healthcheck", false, "run a health check against Anubis")
|
||||
useRemoteAddress = flag.Bool("use-remote-address", false, "read the client's IP address from the network request, useful for debugging and running Anubis on bare metal")
|
||||
healthcheck = flag.Bool("healthcheck", false, "run a health check against nuke")
|
||||
useRemoteAddress = flag.Bool("use-remote-address", false, "read the client's IP address from the network request, useful for debugging and running nuke on bare metal")
|
||||
debugBenchmarkJS = flag.Bool("debug-benchmark-js", false, "respond to every request with a challenge for benchmarking hashrate")
|
||||
ogPassthrough = flag.Bool("og-passthrough", false, "enable Open Graph tag passthrough")
|
||||
ogTimeToLive = flag.Duration("og-expiry-time", 24*time.Hour, "Open Graph tag cache expiration time")
|
||||
ogCacheConsiderHost = flag.Bool("og-cache-consider-host", false, "enable or disable the use of the host in the Open Graph tag cache")
|
||||
extractResources = flag.String("extract-resources", "", "if set, extract the static resources to the specified folder")
|
||||
webmasterEmail = flag.String("webmaster-email", "", "if set, displays webmaster's email on the reject page for appeals")
|
||||
versionFlag = flag.Bool("version", false, "print Anubis version")
|
||||
publicUrl = flag.String("public-url", "", "the externally accessible URL for this Anubis instance, used for constructing redirect URLs (e.g., for forwardAuth).")
|
||||
versionFlag = flag.Bool("version", false, "print nuke version")
|
||||
publicUrl = flag.String("public-url", "", "the externally accessible URL for this nuke instance, used for constructing redirect URLs (e.g., for forwardAuth).")
|
||||
xffStripPrivate = flag.Bool("xff-strip-private", true, "if set, strip private addresses from X-Forwarded-For")
|
||||
customRealIPHeader = flag.String("custom-real-ip-header", "", "if set, read remote IP from header of this name (in case your environment doesn't set X-Real-IP header)")
|
||||
|
||||
thothInsecure = flag.Bool("thoth-insecure", false, "if set, connect to Thoth over plain HTTP/2, don't enable this unless support told you to")
|
||||
thothURL = flag.String("thoth-url", "", "if set, URL for Thoth, the IP reputation database for Anubis")
|
||||
thothToken = flag.String("thoth-token", "", "if set, API token for Thoth, the IP reputation database for Anubis")
|
||||
jwtRestrictionHeader = flag.String("jwt-restriction-header", "X-Real-IP", "If set, the JWT is only valid if the current value of this header matched the value when the JWT was created")
|
||||
)
|
||||
|
||||
|
|
@ -231,7 +227,7 @@ func makeReverseProxy(target string, targetSNI string, targetHost string, insecu
|
|||
return dialer.DialContext(ctx, "unix", addr)
|
||||
}
|
||||
// tell transport how to handle the unix url scheme
|
||||
transport.RegisterProtocol("unix", libanubis.UnixRoundTripper{Transport: transport})
|
||||
transport.RegisterProtocol("unix", libnuke.UnixRoundTripper{Transport: transport})
|
||||
}
|
||||
|
||||
if insecureSkipVerify || targetSNI != "" {
|
||||
|
|
@ -269,14 +265,14 @@ func main() {
|
|||
flag.Parse()
|
||||
|
||||
if *versionFlag {
|
||||
fmt.Println("Anubis", anubis.Version)
|
||||
fmt.Println("Nuke", nuke.Version)
|
||||
return
|
||||
}
|
||||
|
||||
internal.SetHealth("anubis", healthv1.HealthCheckResponse_NOT_SERVING)
|
||||
internal.SetHealth("nuke", healthv1.HealthCheckResponse_NOT_SERVING)
|
||||
|
||||
lg := internal.InitSlog(*slogLevel, os.Stderr)
|
||||
lg.Info("starting up Anubis")
|
||||
lg.Info("starting up Nuke")
|
||||
|
||||
if *healthcheck {
|
||||
log.Println("running healthcheck")
|
||||
|
|
@ -309,7 +305,7 @@ func main() {
|
|||
}
|
||||
|
||||
var rp http.Handler
|
||||
// when using anubis via Systemd and environment variables, then it is not possible to set targe to an empty string but only to space
|
||||
// when using nuke via Systemd and environment variables, then it is not possible to set targe to an empty string but only to space
|
||||
if strings.TrimSpace(*target) != "" {
|
||||
var err error
|
||||
rp, err = makeReverseProxy(*target, *targetSNI, *targetHost, *targetInsecureSkipVerify, *targetDisableKeepAlive)
|
||||
|
|
@ -321,25 +317,8 @@ func main() {
|
|||
if *cookieDomain != "" && *cookieDynamicDomain {
|
||||
log.Fatalf("you can't set COOKIE_DOMAIN and COOKIE_DYNAMIC_DOMAIN at the same time")
|
||||
}
|
||||
|
||||
// Thoth configuration
|
||||
switch {
|
||||
case *thothURL != "" && *thothToken == "":
|
||||
lg.Warn("THOTH_URL is set but no THOTH_TOKEN is set")
|
||||
case *thothURL == "" && *thothToken != "":
|
||||
lg.Warn("THOTH_TOKEN is set but no THOTH_URL is set")
|
||||
case *thothURL != "" && *thothToken != "":
|
||||
lg.Debug("connecting to Thoth")
|
||||
thothClient, err := thoth.New(ctx, *thothURL, *thothToken, *thothInsecure)
|
||||
if err != nil {
|
||||
log.Fatalf("can't dial thoth at %s: %v", *thothURL, err)
|
||||
}
|
||||
|
||||
ctx = thoth.With(ctx, thothClient)
|
||||
}
|
||||
|
||||
lg.Info("loading policy file", "fname", *policyFname)
|
||||
policy, err := libanubis.LoadPoliciesOrDefault(ctx, *policyFname, *challengeDifficulty, *slogLevel)
|
||||
policy, err := libnuke.LoadPoliciesOrDefault(ctx, *policyFname, *challengeDifficulty, *slogLevel)
|
||||
if err != nil {
|
||||
log.Fatalf("can't parse policy file: %v", err)
|
||||
}
|
||||
|
|
@ -351,7 +330,7 @@ func main() {
|
|||
if policy.Store.IsPersistent() {
|
||||
if *hs512Secret == "" && *ed25519PrivateKeyHex == "" && *ed25519PrivateKeyHexFile == "" {
|
||||
lg.Warn("[misconfiguration] persistent storage backend is configured, but no private key is set. " +
|
||||
"Challenges will be invalidated when Anubis restarts. " +
|
||||
"Challenges will be invalidated when nuke restarts. " +
|
||||
"Set HS512_SECRET, ED25519_PRIVATE_KEY_HEX, or ED25519_PRIVATE_KEY_HEX_FILE to ensure challenges survive service restarts. " +
|
||||
"See: https://anubis.techaro.lol/docs/admin/installation#key-generation")
|
||||
}
|
||||
|
|
@ -413,7 +392,7 @@ func main() {
|
|||
log.Fatalf("failed to generate ed25519 key: %v", err)
|
||||
}
|
||||
|
||||
lg.Warn("generating random key, Anubis will have strange behavior when multiple instances are behind the same load balancer target, for more information: see https://anubis.techaro.lol/docs/admin/installation#key-generation")
|
||||
lg.Warn("generating random key, Nuke will have strange behavior when multiple instances are behind the same load balancer target, for more information: see https://anubis.techaro.lol/docs/admin/installation#key-generation")
|
||||
}
|
||||
|
||||
var redirectDomainsList []string
|
||||
|
|
@ -427,13 +406,13 @@ func main() {
|
|||
redirectDomainsList = append(redirectDomainsList, strings.TrimSpace(domain))
|
||||
}
|
||||
} else {
|
||||
lg.Warn("REDIRECT_DOMAINS is not set, Anubis will only redirect to the same domain a request is coming from, see https://anubis.techaro.lol/docs/admin/configuration/redirect-domains")
|
||||
lg.Warn("REDIRECT_DOMAINS is not set, Nuke will only redirect to the same domain a request is coming from, see https://anubis.techaro.lol/docs/admin/configuration/redirect-domains")
|
||||
}
|
||||
|
||||
anubis.CookieName = *cookiePrefix + "-auth"
|
||||
anubis.TestCookieName = *cookiePrefix + "-cookie-verification"
|
||||
anubis.ForcedLanguage = *forcedLanguage
|
||||
anubis.UseSimplifiedExplanation = *useSimplifiedExplanation
|
||||
nuke.CookieName = *cookiePrefix + "-auth"
|
||||
nuke.TestCookieName = *cookiePrefix + "-cookie-verification"
|
||||
nuke.ForcedLanguage = *forcedLanguage
|
||||
nuke.UseSimplifiedExplanation = *useSimplifiedExplanation
|
||||
|
||||
// If OpenGraph configuration values are not set in the config file, use the
|
||||
// values from flags / envvars.
|
||||
|
|
@ -444,7 +423,7 @@ func main() {
|
|||
policy.OpenGraph.Override = map[string]string{}
|
||||
}
|
||||
|
||||
s, err := libanubis.New(libanubis.Options{
|
||||
s, err := libnuke.New(libnuke.Options{
|
||||
BasePrefix: *basePrefix,
|
||||
StripBasePrefix: *stripBasePrefix,
|
||||
Next: rp,
|
||||
|
|
@ -467,11 +446,11 @@ func main() {
|
|||
CookieSameSite: parseSameSite(*cookieSameSite),
|
||||
PublicUrl: *publicUrl,
|
||||
JWTRestrictionHeader: *jwtRestrictionHeader,
|
||||
Logger: policy.Logger.With("subsystem", "anubis"),
|
||||
Logger: policy.Logger.With("subsystem", "nuke"),
|
||||
DifficultyInJWT: *difficultyInJWT,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("can't construct libanubis.Server: %v", err)
|
||||
log.Fatalf("can't construct libnuke.Server: %v", err)
|
||||
}
|
||||
|
||||
var h http.Handler
|
||||
|
|
@ -490,7 +469,7 @@ func main() {
|
|||
"difficulty", *challengeDifficulty,
|
||||
"serveRobotsTXT", *robotsTxt,
|
||||
"target", *target,
|
||||
"version", anubis.Version,
|
||||
"version", nuke.Version,
|
||||
"use-remote-address", *useRemoteAddress,
|
||||
"debug-benchmark-js", *debugBenchmarkJS,
|
||||
"og-passthrough", *ogPassthrough,
|
||||
|
|
@ -510,7 +489,7 @@ func main() {
|
|||
}
|
||||
}()
|
||||
|
||||
internal.SetHealth("anubis", healthv1.HealthCheckResponse_SERVING)
|
||||
internal.SetHealth("nuke", healthv1.HealthCheckResponse_SERVING)
|
||||
|
||||
if err := srv.Serve(listener); !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Fatal(err)
|
||||
|
|
@ -524,9 +503,9 @@ func metricsServer(ctx context.Context, lg slog.Logger, done func()) {
|
|||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", promhttp.Handler())
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
||||
st, ok := internal.GetHealth("anubis")
|
||||
st, ok := internal.GetHealth("nuke")
|
||||
if !ok {
|
||||
slog.Error("health service anubis does not exist, file a bug")
|
||||
slog.Error("health service nuke does not exist, file a bug")
|
||||
}
|
||||
|
||||
switch st {
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
/*
|
||||
Batch process robots.txt files from archives like https://github.com/nrjones8/robots-dot-txt-archive-bot/tree/master/data/cleaned
|
||||
into Anubis CEL policies. Usage: go run batch_process.go <directory with robots.txt files>
|
||||
into nuke CEL policies. Usage: go run batch_process.go <directory with robots.txt files>
|
||||
*/
|
||||
package main
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import (
|
|||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/TecharoHQ/anubis/lib/config"
|
||||
"git.sad.ovh/sophie/nuke/lib/config"
|
||||
|
||||
"sigs.k8s.io/yaml"
|
||||
)
|
||||
|
|
@ -36,7 +36,7 @@ type RobotsRule struct {
|
|||
IsBlacklist bool // true if this is a specifically denied user agent
|
||||
}
|
||||
|
||||
type AnubisRule struct {
|
||||
type NukeRule struct {
|
||||
Expression *config.ExpressionOrList `yaml:"expression,omitempty" json:"expression,omitempty"`
|
||||
Challenge *config.ChallengeRules `yaml:"challenge,omitempty" json:"challenge,omitempty"`
|
||||
Weight *config.Weight `yaml:"weight,omitempty" json:"weight,omitempty"`
|
||||
|
|
@ -95,11 +95,11 @@ func main() {
|
|||
log.Fatalf("failed to parse robots.txt: %v", err)
|
||||
}
|
||||
|
||||
// Convert to Anubis rules
|
||||
anubisRules := convertToAnubisRules(rules)
|
||||
// Convert to Nuke rules
|
||||
nukeRules := convertToNukeRules(rules)
|
||||
|
||||
// Check if any rules were generated
|
||||
if len(anubisRules) == 0 {
|
||||
if len(nukeRules) == 0 {
|
||||
log.Fatal("no valid rules generated from robots.txt - file may be empty or contain no disallow directives")
|
||||
}
|
||||
|
||||
|
|
@ -107,9 +107,9 @@ func main() {
|
|||
var output []byte
|
||||
switch strings.ToLower(*outputFormat) {
|
||||
case "yaml":
|
||||
output, err = yaml.Marshal(anubisRules)
|
||||
output, err = yaml.Marshal(nukeRules)
|
||||
case "json":
|
||||
output, err = json.MarshalIndent(anubisRules, "", " ")
|
||||
output, err = json.MarshalIndent(nukeRules, "", " ")
|
||||
default:
|
||||
log.Fatalf("unsupported output format: %s (use yaml or json)", *outputFormat)
|
||||
}
|
||||
|
|
@ -126,7 +126,7 @@ func main() {
|
|||
if err != nil {
|
||||
log.Fatalf("failed to write output file: %v", err)
|
||||
}
|
||||
fmt.Printf("Generated Anubis policy written to %s\n", *outputFile)
|
||||
fmt.Printf("Generated Nuke policy written to %s\n", *outputFile)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -227,8 +227,8 @@ func parseIntSafe(s string) (int, error) {
|
|||
return result, err
|
||||
}
|
||||
|
||||
func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
|
||||
var anubisRules []AnubisRule
|
||||
func convertToNukeRules(robotsRules []RobotsRule) []NukeRule {
|
||||
var nukeRules []NukeRule
|
||||
ruleCounter := 0
|
||||
|
||||
// Process each robots rule individually
|
||||
|
|
@ -238,7 +238,7 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
|
|||
// Handle crawl delay
|
||||
if robotsRule.CrawlDelay > 0 && *crawlDelay > 0 {
|
||||
ruleCounter++
|
||||
rule := AnubisRule{
|
||||
rule := NukeRule{
|
||||
Name: fmt.Sprintf("%s-crawl-delay-%d", *policyName, ruleCounter),
|
||||
Action: "WEIGH",
|
||||
Weight: &config.Weight{Adjust: *crawlDelay},
|
||||
|
|
@ -266,13 +266,13 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
|
|||
Any: expressions,
|
||||
}
|
||||
}
|
||||
anubisRules = append(anubisRules, rule)
|
||||
nukeRules = append(nukeRules, rule)
|
||||
}
|
||||
|
||||
// Handle blacklisted user agents
|
||||
if robotsRule.IsBlacklist {
|
||||
ruleCounter++
|
||||
rule := AnubisRule{
|
||||
rule := NukeRule{
|
||||
Name: fmt.Sprintf("%s-blacklist-%d", *policyName, ruleCounter),
|
||||
Action: *userAgentDeny,
|
||||
}
|
||||
|
|
@ -306,7 +306,7 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
|
|||
Any: expressions,
|
||||
}
|
||||
}
|
||||
anubisRules = append(anubisRules, rule)
|
||||
nukeRules = append(nukeRules, rule)
|
||||
}
|
||||
|
||||
// Handle specific disallow rules
|
||||
|
|
@ -316,7 +316,7 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
|
|||
}
|
||||
|
||||
ruleCounter++
|
||||
rule := AnubisRule{
|
||||
rule := NukeRule{
|
||||
Name: fmt.Sprintf("%s-disallow-%d", *policyName, ruleCounter),
|
||||
Action: *baseAction,
|
||||
}
|
||||
|
|
@ -338,7 +338,7 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
|
|||
continue // Skip wildcard as it's handled separately
|
||||
}
|
||||
ruleCounter++
|
||||
subRule := AnubisRule{
|
||||
subRule := NukeRule{
|
||||
Name: fmt.Sprintf("%s-disallow-%d", *policyName, ruleCounter),
|
||||
Action: *baseAction,
|
||||
Expression: &config.ExpressionOrList{
|
||||
|
|
@ -348,7 +348,7 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
|
|||
},
|
||||
},
|
||||
}
|
||||
anubisRules = append(anubisRules, subRule)
|
||||
nukeRules = append(nukeRules, subRule)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
|
@ -361,11 +361,11 @@ func convertToAnubisRules(robotsRules []RobotsRule) []AnubisRule {
|
|||
All: conditions,
|
||||
}
|
||||
|
||||
anubisRules = append(anubisRules, rule)
|
||||
nukeRules = append(nukeRules, rule)
|
||||
}
|
||||
}
|
||||
|
||||
return anubisRules
|
||||
return nukeRules
|
||||
}
|
||||
|
||||
func buildPathCondition(robotsPath string) string {
|
||||
|
|
|
|||
|
|
@ -136,16 +136,16 @@ func TestDataFileConversion(t *testing.T) {
|
|||
*userAgentDeny = oldDeniedAction
|
||||
}()
|
||||
|
||||
// Convert to Anubis rules
|
||||
anubisRules := convertToAnubisRules(rules)
|
||||
// Convert to Nuke rules
|
||||
nukeRules := convertToNukeRules(rules)
|
||||
|
||||
// Generate output
|
||||
var actualOutput []byte
|
||||
switch strings.ToLower(*outputFormat) {
|
||||
case "yaml":
|
||||
actualOutput, err = yaml.Marshal(anubisRules)
|
||||
actualOutput, err = yaml.Marshal(nukeRules)
|
||||
case "json":
|
||||
actualOutput, err = json.MarshalIndent(anubisRules, "", " ")
|
||||
actualOutput, err = json.MarshalIndent(nukeRules, "", " ")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal output: %v", err)
|
||||
|
|
@ -249,10 +249,10 @@ Disallow: /admin`
|
|||
*policyName = "test-policy"
|
||||
defer func() { *policyName = oldPolicyName }()
|
||||
|
||||
anubisRules := convertToAnubisRules(rules)
|
||||
nukeRules := convertToNukeRules(rules)
|
||||
|
||||
// Test YAML output
|
||||
yamlOutput, err := yaml.Marshal(anubisRules)
|
||||
yamlOutput, err := yaml.Marshal(nukeRules)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal YAML: %v", err)
|
||||
}
|
||||
|
|
@ -262,7 +262,7 @@ Disallow: /admin`
|
|||
}
|
||||
|
||||
// Test JSON output
|
||||
jsonOutput, err := json.MarshalIndent(anubisRules, "", " ")
|
||||
jsonOutput, err := json.MarshalIndent(nukeRules, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal JSON: %v", err)
|
||||
}
|
||||
|
|
@ -290,14 +290,14 @@ Disallow: /admin`
|
|||
*baseAction = action
|
||||
defer func() { *baseAction = oldAction }()
|
||||
|
||||
anubisRules := convertToAnubisRules(rules)
|
||||
nukeRules := convertToNukeRules(rules)
|
||||
|
||||
if len(anubisRules) != 1 {
|
||||
t.Fatalf("Expected 1 rule, got %d", len(anubisRules))
|
||||
if len(nukeRules) != 1 {
|
||||
t.Fatalf("Expected 1 rule, got %d", len(nukeRules))
|
||||
}
|
||||
|
||||
if anubisRules[0].Action != action {
|
||||
t.Errorf("Expected action %s, got %s", action, anubisRules[0].Action)
|
||||
if nukeRules[0].Action != action {
|
||||
t.Errorf("Expected action %s, got %s", action, nukeRules[0].Action)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -325,10 +325,10 @@ Disallow: /`
|
|||
*policyName = name
|
||||
defer func() { *policyName = oldName }()
|
||||
|
||||
anubisRules := convertToAnubisRules(rules)
|
||||
nukeRules := convertToNukeRules(rules)
|
||||
|
||||
// Check that all rule names use the custom prefix
|
||||
for _, rule := range anubisRules {
|
||||
for _, rule := range nukeRules {
|
||||
if !strings.HasPrefix(rule.Name, name+"-") {
|
||||
t.Errorf("Rule name %s doesn't start with expected prefix %s-", rule.Name, name)
|
||||
}
|
||||
|
|
@ -360,11 +360,11 @@ Crawl-delay: 60`
|
|||
*crawlDelay = weight
|
||||
defer func() { *crawlDelay = oldWeight }()
|
||||
|
||||
anubisRules := convertToAnubisRules(rules)
|
||||
nukeRules := convertToNukeRules(rules)
|
||||
|
||||
// Count weight rules and verify they have correct weight
|
||||
weightRules := 0
|
||||
for _, rule := range anubisRules {
|
||||
for _, rule := range nukeRules {
|
||||
if rule.Action == "WEIGH" && rule.Weight != nil {
|
||||
weightRules++
|
||||
if rule.Weight.Adjust != weight {
|
||||
|
|
@ -402,10 +402,10 @@ Disallow: /`
|
|||
*userAgentDeny = action
|
||||
defer func() { *userAgentDeny = oldAction }()
|
||||
|
||||
anubisRules := convertToAnubisRules(rules)
|
||||
nukeRules := convertToNukeRules(rules)
|
||||
|
||||
// All rules should be blacklist rules with the specified action
|
||||
for _, rule := range anubisRules {
|
||||
for _, rule := range nukeRules {
|
||||
if !strings.Contains(rule.Name, "blacklist") {
|
||||
t.Errorf("Expected blacklist rule, got %s", rule.Name)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue