test: introduce SSH based CI for non-native test hosts (#644)

* feat: ssh based CI

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test: implement SSH ci with caches and github actions

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): fix known hosts secret

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): clone the repo, that's important

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): speed up ci by prebaking the SSH CI image

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): set -euo

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): enable pull_request_target so things work

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): oh goody it's broken

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): add cronjob to rebuild ci runner image

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): also run yeet

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): force git version for yeet

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): run set -x in the container

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): fix yeet?

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): remove yeet for now

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(ssh-ci): disable for PRs for now

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/expect.txt

@@ -6,6 +6,7 @@ amazonbot
 anthro
 anubis
 anubistest
+apk
 Applebot
 archlinux
 badregexes
@@ -68,6 +69,7 @@ duckduckbot
 eerror
 ellenjoe
 enbyware
+euo
 everyones
 evilbot
 evilsite
@@ -117,6 +119,7 @@ imgproxy
 inp
 iss
 isset
+itv
 ivh
 Jenomis
 JGit
@@ -246,6 +249,7 @@ traefik
 uberspace
 unixhttpd
 unmarshal
+uuidgen
 uvx
 UXP
 Varis

.github/workflows/docker.yml

@@ -3,8 +3,8 @@ name: Docker image builds
 on:
   workflow_dispatch:
   push:
-    branches: [ "main" ]
-    tags: [ "v*" ]
+    branches: ["main"]
+    tags: ["v*"]
 
 env:
   DOCKER_METADATA_SET_OUTPUT_ENV: "true"
@@ -55,7 +55,7 @@ jobs:
         run: |
           brew bundle
 
-      - name: Log into registry 
+      - name: Log into registry
         uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
@@ -77,7 +77,6 @@ jobs:
           DOCKER_REPO: ${{ env.IMAGE }}
           SLOG_LEVEL: debug
 
-      
       - name: Generate artifact attestation
         uses: actions/attest-build-provenance@db473fddc028af60658334401dc6fa3ffd8669fd # v2.3.0
         with:

.github/workflows/ssh-ci-runner-cron.yml

@@ -0,0 +1,36 @@
+name: Regenerate ssh ci runner image
+
+on:
+  # pull_request:
+  #   branches: ["main"]
+  schedule:
+    - cron: "0 0 1,8,15,22 * *"
+  workflow_dispatch:
+
+permissions:
+  pull-requests: write
+  contents: write
+  packages: write
+
+jobs:
+  ssh-ci-rebuild:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Log into registry
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+      - name: Build and push
+        run: |
+          cd ./test/ssh-ci
+          docker buildx bake --push

.github/workflows/ssh-ci.yml

@@ -0,0 +1,36 @@
+name: SSH CI
+
+on:
+  push:
+    branches: ["main"]
+  # pull_request:
+  #   branches: ["main"]
+
+permissions:
+  contents: read
+
+jobs:
+  ssh:
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        host:
+          - ubuntu@riscv64.techaro.lol
+          - ci@ppc64le.techaro.lol
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Install CI target SSH key
+        uses: shimataro/ssh-key-action@d4fffb50872869abe2d9a9098a6d9c5aa7d16be4 # v2.7.0
+        with:
+          key: ${{ secrets.CI_SSH_KEY }}
+          name: id_rsa
+          known_hosts: ${{ secrets.CI_SSH_KNOWN_HOSTS }}
+      - name: Run CI
+        run: bash test/ssh-ci/rigging.sh ${{ matrix.host }}
+        env:
+          GITHUB_RUN_ID: ${{ github.run_id }}