fix(lib): enable multiple consecutive slash support (#1155)

* fix(lib): enable multiple consecutive slash support

Closes #754
Closes #808
Closes #815

Apparently more applications use multiple slashes in a row than I
thought. There is no easy way around this other than to do this hacky
fix to avoid net/http#ServeMux's URL cleaning.

* test(double_slash): add sourceware case

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(lib): fix tests for double slash fix

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <xe.iaso@techaro.lol>
Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/workflows/smoke-tests.yml

@@ -14,6 +14,7 @@ jobs:
     strategy:
       matrix:
         test:
+          - double_slash
           - forced-language
           - git-clone
           - git-push

docs/docs/CHANGELOG.md

@@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixes concurrency problems with very old browsers ([#1082](https://github.com/TecharoHQ/anubis/issues/1082)).
 - Randomly use the Refresh header instead of the meta refresh tag in the metarefresh challenge.
 - Update OpenRC service to truncate the runtime directory before starting Anubis.
+- Allow multiple consecutive slashes in a row in application paths ([#754](https://github.com/TecharoHQ/anubis/issues/754)).
 - Add option to set `targetSNI` to special keyword 'auto' to indicate that it should be automatically set to the request Host name ([424](https://github.com/TecharoHQ/anubis/issues/424)).
 
 ### Bug Fixes

lib/anubis_test.go

@@ -457,7 +457,7 @@ func TestBasePrefix(t *testing.T) {
 	}{
 		{
 			name:       "no prefix",
-			basePrefix: "/",
+			basePrefix: "",
 			path:       "/.within.website/x/cmd/anubis/api/make-challenge",
 			expected:   "/.within.website/x/cmd/anubis/api/make-challenge",
 		},
@@ -499,9 +499,15 @@ func TestBasePrefix(t *testing.T) {
 			}
 
 			q := req.URL.Query()
-			q.Set("redir", tc.basePrefix)
+			redir := tc.basePrefix
+			if tc.basePrefix == "" {
+				redir = "/"
+			}
+			q.Set("redir", redir)
 			req.URL.RawQuery = q.Encode()
 
+			t.Log(req.URL.String())
+
 			// Test API endpoint with prefix
 			resp, err := cli.Do(req)
 			if err != nil {
@@ -513,8 +519,15 @@ func TestBasePrefix(t *testing.T) {
 				t.Errorf("expected status code %d, got: %d", http.StatusOK, resp.StatusCode)
 			}
 
+			data, err := io.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("can't read body: %v", err)
+			}
+
+			t.Log(string(data))
+
 			var chall challengeResp
-			if err := json.NewDecoder(resp.Body).Decode(&chall); err != nil {
+			if err := json.NewDecoder(bytes.NewBuffer(data)).Decode(&chall); err != nil {
 				t.Fatalf("can't read challenge response body: %v", err)
 			}
 
@@ -535,7 +548,7 @@ func TestBasePrefix(t *testing.T) {
 				nonce++
 			}
 			elapsedTime := 420
-			redir := "/"
+			redir = "/"
 
 			cli.CheckRedirect = func(req *http.Request, via []*http.Request) error {
 				return http.ErrUseLastResponse

lib/config.go

@@ -107,7 +107,7 @@ func New(opts Options) (*Server, error) {
 		opts.ED25519PrivateKey = priv
 	}
 
-	anubis.BasePrefix = opts.BasePrefix
+	anubis.BasePrefix = strings.TrimRight(opts.BasePrefix, "/")
 	anubis.PublicUrl = opts.PublicUrl
 
 	result := &Server{

lib/http.go

@@ -279,7 +279,12 @@ func (s *Server) respondWithStatus(w http.ResponseWriter, r *http.Request, msg s
 }
 
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	s.mux.ServeHTTP(w, r)
+	if strings.HasPrefix(r.URL.Path, anubis.BasePrefix+anubis.StaticPath) {
+		s.mux.ServeHTTP(w, r)
+		return
+	}
+
+	s.maybeReverseProxyOrPage(w, r)
 }
 
 func (s *Server) stripBasePrefixFromRequest(r *http.Request) *http.Request {

test/cmd/httpdebug/main.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"log/slog"
+	"net/http"
+)
+
+var (
+	bind = flag.String("bind", ":3923", "TCP port to bind to")
+)
+
+func main() {
+	flag.Parse()
+
+	slog.Info("listening", "url", "http://localhost"+*bind)
+	log.Fatal(http.ListenAndServe(*bind, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		slog.Info("got request", "method", r.Method, "path", r.RequestURI)
+
+		fmt.Fprintln(w, r.Method, r.RequestURI)
+		r.Header.Write(w)
+	})))
+}

test/double_slash/anubis.yaml

@@ -0,0 +1,8 @@
+bots:
+  - name: challenge
+    user_agent_regex: CHALLENGE
+    action: CHALLENGE
+
+status_codes:
+  CHALLENGE: 200
+  DENY: 403

test/double_slash/input.txt

@@ -0,0 +1,178 @@
+/wiki//bin
+/wiki//boot
+/wiki//dev
+/wiki//dev/de
+/wiki//dev/en
+/wiki//dev/en-ca
+/wiki//dev/es
+/wiki//dev/fr
+/wiki//dev/hr
+/wiki//dev/hu
+/wiki//dev/it
+/wiki//dev/ja
+/wiki//dev/ko
+/wiki//dev/pl
+/wiki//dev/pt-br
+/wiki//dev/ro
+/wiki//dev/ru
+/wiki//dev/sv
+/wiki//dev/uk
+/wiki//dev/zh-cn
+/wiki//etc
+/wiki//etc/conf.d
+/wiki//etc/env.d
+/wiki//etc/fstab
+/wiki//etc/fstab/de
+/wiki//etc/fstab/en
+/wiki//etc/fstab/es
+/wiki//etc/fstab/fr
+/wiki//etc/fstab/hu
+/wiki//etc/fstab/it
+/wiki//etc/fstab/ja
+/wiki//etc/fstab/ko
+/wiki//etc/fstab/ru
+/wiki//etc/fstab/sv
+/wiki//etc/fstab/uk
+/wiki//etc/fstab/zh-cn
+/wiki//etc/hosts
+/wiki//etc/local.d
+/wiki//etc/make.conf
+/wiki//etc/portage
+/wiki//etc/portage/bashrc
+/wiki//etc/portage/Bashrc
+/wiki//etc/portage/binrepos.conf
+/wiki//etc/portage/binrepos.conf/en
+/wiki//etc/portage/binrepos.conf/hu
+/wiki//etc/portage/binrepos.conf/ja
+/wiki//etc/portage/binrepos.conf/ru
+/wiki//etc/portage/categories
+/wiki//etc/portage/color.map
+/wiki//etc/portage/env
+/wiki//etc/portage/img/ico.png
+/wiki//etc/portage/license_groups
+/wiki//etc/portage/make.conf
+/wiki//etc/portage/make.conf/de
+/wiki//etc/portage/make.conf/de/etc/portage/make.conf
+/wiki//etc/portage/make.conf/en
+/wiki//etc/portage/make.conf/es
+/wiki//etc/portage/make.conf/fr
+/wiki//etc/portage/make.conf/hu
+/wiki//etc/portage/make.conf/it
+/wiki//etc/portage/make.conf/it/var/db/repos/gentoo/licenses
+/wiki//etc/portage/make.conf/ja
+/wiki//etc/portage/make.conf/pl
+/wiki//etc/portage/make.conf/ru
+/wiki//etc/portage/make.conf/uk
+/wiki//etc/portage/make.conf/zh-cn
+/wiki//etc/portage/make.profile
+/wiki//etc/portage/mirrors
+/wiki//etc/portage/modules
+/wiki//etc/portage/package.accept_keywords
+/wiki//etc/portage/package.env
+/wiki//etc/portage/package.license
+/wiki//etc/portage/package.license/en
+/wiki//etc/portage/package.license/es
+/wiki//etc/portage/package.license/hu
+/wiki//etc/portage/package.license/ja
+/wiki//etc/portage/package.mask
+/wiki//etc/portage/package.mask/en
+/wiki//etc/portage/package.mask/hu
+/wiki//etc/portage/package.mask/ja
+/wiki//etc/portage/package.properties
+/wiki//etc/portage/package.unmask
+/wiki//etc/portage/package.use
+/wiki//etc/portage/package.use/de
+/wiki//etc/portage/package.use/en
+/wiki//etc/portage/package.use/es
+/wiki//etc/portage/package.use/fr
+/wiki//etc/portage/package.use/hu
+/wiki//etc/portage/package.use/it
+/wiki//etc/portage/package.use/ja
+/wiki//etc/portage/package.use/ru
+/wiki//etc/portage/package.use/uk
+/wiki//etc/portage/package.use/zh-cn
+/wiki//etc/portage/patches
+/wiki//etc/portage/profile/make.defaults
+/wiki//etc/portage/profile/package.provided
+/wiki//etc/portage/profile/package.provided/etc/portage/profile/package.provided
+/wiki//etc/portage/profile/package.provided/etc/portage/profiles/package.provided
+/wiki//etc/portage/profile/package.use.mask
+/wiki//etc/portage/profiles/package.provided
+/wiki//etc/portage/profiles/package.use.mask
+/wiki//etc/portage/profiles/package.use.mask/etc/portage/profile/package.use.mask
+/wiki//etc/portage/profiles/package.use.mask/etc/portage/profiles/package.use.mask
+/wiki//etc/portage/profiles/use.mask
+/wiki//etc/portage/profile/use.mask
+/wiki//etc/portage/repos.conf
+/wiki//etc/portage/repos.conf/brother-overlay.conf
+/wiki//etc/portage/repos.conf/de
+/wiki//etc/portage/repos.conf/en
+/wiki//etc/portage/repos.conf/es
+/wiki//etc/portage/repos.conf/etc/portage/repos.conf/gentoo.conf
+/wiki//etc/portage/repos.conf/fr
+/wiki//etc/portage/repos.conf/fr/etc/portage/repos.conf/gentoo.conf
+/wiki//etc/portage/repos.conf/gentoo.conf
+/wiki//etc/portage/repos.conf/gentoo.conf/etc/portage/repos.conf/gentoo.conf
+/wiki//etc/portage/repos.conf/hr
+/wiki//etc/portage/repos.conf/hu
+/wiki//etc/portage/repos.conf/it
+/wiki//etc/portage/repos.conf/ja
+/wiki//etc/portage/repos.conf/ko
+/wiki//etc/portage/repos.conf/pl
+/wiki//etc/portage/repos.conf/pt-br
+/wiki//etc/portage/repos.conf/ru
+/wiki//etc/portage/repos.conf/uk
+/wiki//etc/portage/repos.conf/zh-cn
+/wiki//etc/portage/savedconfig
+/wiki//etc/portage/sets
+/wiki//etc/profile
+/wiki//etc/profile.env
+/wiki//etc/sandbox.conf
+/wiki//home
+/wiki//lib
+/wiki//lib64
+/wiki//media
+/wiki//mnt
+/wiki//opt
+/wiki//proc
+/wiki//proc/config.gz
+/wiki//run
+/wiki//sbin
+/wiki//srv
+/wiki//sys
+/wiki//tmp
+/wiki//usr
+/wiki//usr/bin
+/wiki//usr_move
+/wiki//usr/portage
+/wiki//usr/portage/distfiles
+/wiki//usr/portage/licenses
+/wiki//usr/portage/metadata
+/wiki//usr/portage/metadata/md5-cache
+/wiki//usr/portage/metadata/md5-cache/usr/portage/metadata/md5-cache
+/wiki//usr/portage/metadata/md5-cache/var/db/repos/gentoo//metadata/md5-cache
+/wiki//usr/portage/packages
+/wiki//usr/portage/profiles
+/wiki//usr/portage/profiles/license_groups
+/wiki//usr/portage/profiles/license_groups/usr/portage/profiles/license_groups
+/wiki//usr/portage/profiles/license_groups/var/db/repos/gentoo//profiles/license_groups
+/wiki//usr/share/doc/
+/wiki//var/cache/binpkgs
+/wiki//var/cache/distfiles
+/wiki//var/db/pkg
+/wiki//var/db/pkg%22
+/wiki//var/db/repos/gentoo
+/wiki//var/db/repos/gentoo/licenses
+/wiki//var/db/repos/gentoo/licenses/var/db/repos/gentoo//licenses
+/wiki//var/db/repos/gentoo/licenses/var/db/repos/gentoo/licenses
+/wiki//var/db/repos/gentoo/metadata
+/wiki//var/db/repos/gentoo/metadata/md5-cache
+/wiki//var/db/repos/gentoo/metadata/var/db/repos/gentoo//metadata
+/wiki//var/db/repos/gentoo/metadata/var/db/repos/gentoo/metadata
+/wiki//var/db/repos/gentoo/profiles
+/wiki//var/db/repos/gentoo/profiles/license_groups
+/wiki//var/db/repos/gentoo/profiles/package.mask
+/wiki//var/lib/portage
+/wiki//var/lib/portage/world
+/wiki//var/run
+/gcc-bugs/bug-122002-4@http.gcc.gnu.org%2Fbugzilla%2F/T/

test/double_slash/test.mjs

@@ -0,0 +1,45 @@
+import { createReadStream } from "fs";
+import { createInterface } from "readline";
+
+async function getPage(path) {
+  return fetch(`http://localhost:8923${path}`)
+    .then(resp => {
+      if (resp.status !== 200) {
+        throw new Error(`wanted status 200, got status: ${resp.status}`);
+      }
+      return resp;
+    })
+    .then(resp => resp.text());
+}
+
+(async () => {
+  const fin = createReadStream("input.txt");
+  const rl = createInterface({
+    input: fin,
+    crlfDelay: Infinity,
+  });
+
+  const resultSheet = {};
+
+  let failed = false;
+
+  for await (const line of rl) {
+    console.log(line);
+
+    const resp = await getPage(line);
+    resultSheet[line] = {
+      match: resp.includes(`GET ${line}`),
+      line: resp.split("\n")[0],
+    };
+  }
+
+  for (let [k, v] of Object.entries(resultSheet)) {
+    if (!v.match) {
+      failed = true;
+    }
+
+    console.debug({ path: k, results: v });
+  }
+
+  process.exit(failed ? 1 : 0);
+})();

test/double_slash/test.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+function cleanup() {
+	pkill -P $$
+}
+
+trap cleanup EXIT SIGINT
+
+# Build static assets
+(cd ../.. && npm ci && npm run assets)
+
+go tool anubis --help 2>/dev/null || :
+
+go run ../cmd/httpdebug &
+
+go tool anubis \
+	--policy-fname ./anubis.yaml \
+	--use-remote-address \
+	--target=http://localhost:3923 &
+
+backoff-retry node ./test.mjs

test/double_slash/var/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore