deep-test.yml

name: Deep test

# Manually-dispatched full-platform validation, beyond what the regular CI job
# covers: the OS service lifecycle (install -> auto-start -> probes -> stop /
# restart -> uninstall), data-dir permissions, netinfo/traceroute paths, and the
# Docker image + the canonical compose flow. Dispatch after platform-sensitive
# changes; artifacts carry assertion output and diagnostic log tails per leg.
on:
  workflow_dispatch:
  schedule:
    # Saturdays 05:37 UTC - a slot of its own, clear of ci.yml's Monday vuln
    # re-scan and the fleet probe's Thursday, so service-lifecycle drift is
    # never tangled with either's noise. Dispatch-only coverage rotted quietly:
    # the best platform validation in the pipeline only ran when someone
    # remembered to ask for it.
    - cron: '37 5 * * 6'

permissions:
  contents: read

jobs:
  windows:
    runs-on: windows-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
        with:
          go-version-file: go.mod

      - name: full test suite
        shell: bash
        run: go test ./... -count=1 2>&1 | tee full-test.log

      - name: build
        run: go build -o pingularity.exe .

      - name: service lifecycle (install auto-starts, probes, stop, restart, uninstall)
        shell: pwsh
        run: |
          # Failures are counted, then asserted at the end - never thrown.
          # $ErrorActionPreference stays 'Continue' (overriding the 'stop' the
          # runner prepends) so one dead probe cannot end the script before the
          # log is complete, and Tee-Object is a cmdlet that always succeeds,
          # so no native exit code ever reaches the step on its own. Unlike the
          # launchd leg there is no subshell here, so a plain counter survives
          # to the final assertion.
          $ErrorActionPreference = 'Continue'
          $script:fails = 0
          function Note($msg) {
            "ASSERT FAIL: $msg" | Tee-Object svc.log -Append
            $script:fails += 1
          }
          # Judges the exit code of the native command that just ran; Tee-Object
          # and Start-Sleep are cmdlets and do not touch $LASTEXITCODE, but call
          # this straight after the pipeline it belongs to anyway.
          function Must($what) { if ($LASTEXITCODE -ne 0) { Note "$what exited $LASTEXITCODE" } }
          function Probe($what, $url) {
            try { $r = Invoke-WebRequest -UseBasicParsing -TimeoutSec 5 $url
                  "PROBE OK  $what -> $($r.StatusCode)" | Tee-Object svc.log -Append
                  if ($r.StatusCode -ne 200) { Note "$what returned $($r.StatusCode), want 200" } }
            catch { "PROBE FAIL $what -> $($_.Exception.Message)" | Tee-Object svc.log -Append
                    Note "$what did not return 200" }
          }
          "=== install (must register AND start) ===" | Tee-Object svc.log -Append
          .\pingularity.exe install 2>&1 | Tee-Object svc.log -Append
          Must "install"
          Start-Sleep 8
          .\pingularity.exe status 2>&1 | Tee-Object svc.log -Append
          Must "status"
          foreach ($i in 1..15) { try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 3 http://127.0.0.1:9000/metrics | Out-Null; break } catch { Start-Sleep 2 } }
          Probe "metrics"   http://127.0.0.1:9000/metrics
          Probe "status"    http://127.0.0.1:9000/api/status
          Probe "dashboard" http://127.0.0.1:9000/
          try { Invoke-WebRequest -UseBasicParsing -Method POST -ContentType 'application/json' -TimeoutSec 10 http://127.0.0.1:9000/api/netinfo | Out-Null } catch { Note "netinfo POST failed" }
          Start-Sleep 20
          try { (Invoke-WebRequest -UseBasicParsing -TimeoutSec 5 http://127.0.0.1:9000/api/netinfo).Content | Tee-Object svc.log -Append } catch { Note "netinfo GET failed" }
          "=== stop / status / start / status ===" | Tee-Object svc.log -Append
          .\pingularity.exe stop    2>&1 | Tee-Object svc.log -Append
          Must "stop"
          Start-Sleep 3
          # `status` exits 0 whether running, stopped or not installed, so the
          # port going quiet is the assertion here, not an exit code.
          .\pingularity.exe status  2>&1 | Tee-Object svc.log -Append
          try { Invoke-WebRequest -UseBasicParsing -TimeoutSec 3 http://127.0.0.1:9000/metrics | Out-Null
                Note "the dashboard still answered after stop" } catch {}
          .\pingularity.exe start   2>&1 | Tee-Object svc.log -Append
          Must "start"
          Start-Sleep 5
          .\pingularity.exe status  2>&1 | Tee-Object svc.log -Append
          Must "status after restart"
          Probe "metrics-after-restart" http://127.0.0.1:9000/metrics
          "=== data dir + DACL (want owner-only: no Users/Everyone entry) ===" | Tee-Object svc.log -Append
          # The Windows analogue of the launchd leg's 700/600 check: osperm sets
          # a protected DACL naming only the service account, SYSTEM and
          # Administrators, so a broad principal on the data dir or the database
          # means every local user can read the monitoring history.
          $dirAcl = icacls C:\ProgramData\pingularity 2>&1
          Must "icacls data dir"
          $dirAcl | Tee-Object svc.log -Append
          $dbAcl = icacls C:\ProgramData\pingularity\pingularity.db 2>&1
          Must "icacls database"
          $dbAcl | Tee-Object svc.log -Append
          $broad = @($dirAcl) + @($dbAcl) | Select-String 'Everyone|BUILTIN\\Users|Authenticated Users'
          if ($broad) { Note "the DACL grants a broad principal: $(($broad.Line.Trim()) -join '; ')" }
          .\pingularity.exe reset-auth 2>&1 | Tee-Object svc.log -Append
          Must "reset-auth"
          .\pingularity.exe uninstall -y 2>&1 | Tee-Object svc.log -Append
          Must "uninstall"
          Start-Sleep 3
          .\pingularity.exe status 2>&1 | Tee-Object svc.log -Append
          if (Test-Path C:\ProgramData\pingularity\pingularity.db) {
            "db survives uninstall: True" | Tee-Object svc.log -Append
          } else {
            Note "uninstall deleted the database - removing the service must not take the user's history with it"
          }
          if ($script:fails -ne 0) { "::error::$script:fails assertion(s) failed in the service lifecycle - see svc.log"; exit 1 }

      - name: upload logs
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: deep-test-windows
          path: |
            full-test.log
            svc.log

  macos:
    runs-on: macos-latest
    timeout-minutes: 30
    # `shell: bash` is what selects `bash -eo pipefail`; without it GitHub runs
    # `bash -e`, where `cmd | tee log` reports tee's exit status and a failing
    # command inside the pipeline passes silently. Every block below pipes to tee.
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
        with:
          go-version-file: go.mod

      - name: full test suite
        run: go test ./... -count=1 2>&1 | tee full-test.log

      - name: build
        run: go build -o pingularity .

      - name: service lifecycle (launchd; install auto-starts, probes, stop, restart, uninstall)
        run: |
          # Failures are counted in a FILE. The block below is the left side of a
          # pipe, so it runs in a subshell and a shell variable would not survive
          # back out to the step's exit status - the step would pass while the log
          # it just wrote was full of failures.
          FAILS=$(mktemp)
          note() { echo "ASSERT FAIL: $*"; echo 1 >> "$FAILS"; }
          probe() {
            code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "$2" || echo FAIL)
            echo "PROBE $1 -> $code"
            [ "$code" = "200" ] || note "$1 returned $code, want 200"
          }
          DIR="/Library/Application Support/pingularity"
          {
            echo "=== install (must register AND start) ==="
            sudo ./pingularity install
            sleep 8
            sudo ./pingularity status
            for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9000/metrics >/dev/null 2>&1 && break; sleep 2; done
            probe metrics   http://127.0.0.1:9000/metrics
            probe status    http://127.0.0.1:9000/api/status
            probe dashboard http://127.0.0.1:9000/
            echo "=== netinfo refresh (macOS ICMP-socket traceroute path) ==="
            curl -fsS -X POST -H 'Content-Type: application/json' --max-time 10 http://127.0.0.1:9000/api/netinfo >/dev/null 2>&1 \
              || note "netinfo POST failed"
            sleep 20
            curl -fsS --max-time 5 http://127.0.0.1:9000/api/netinfo || note "netinfo GET failed"
            echo
            echo "=== stop / status / start / status ==="
            sudo ./pingularity stop;  sleep 3
            # `status` reports NOT running here, so its exit code is not the
            # assertion - the port going quiet is.
            sudo ./pingularity status || true
            curl -fsS --max-time 3 http://127.0.0.1:9000/metrics >/dev/null 2>&1 \
              && note "the dashboard still answered after stop"
            sudo ./pingularity start; sleep 5
            sudo ./pingularity status
            probe metrics-after-restart http://127.0.0.1:9000/metrics
            echo "=== data dir perms (root daemon path, want 700/600) ==="
            sudo ls -la "$DIR" || true
            dperm=$(sudo stat -f '%Sp' "$DIR")
            fperm=$(sudo stat -f '%Sp' "$DIR/pingularity.db")
            echo "perms: dir=$dperm db=$fperm"
            [ "$dperm" = "drwx------" ] || note "data dir is $dperm, want drwx------"
            [ "$fperm" = "-rw-------" ] || note "database is $fperm, want -rw-------"
            echo "=== reset-auth ==="
            sudo ./pingularity reset-auth
            echo "=== uninstall -y ==="
            sudo ./pingularity uninstall -y
            sleep 3
            sudo ./pingularity status || true   # not installed: non-zero is correct
            if sudo test -f "$DIR/pingularity.db"; then
              echo "db survives uninstall: True"
            else
              note "uninstall deleted the database - removing the service must not take the user's history with it"
            fi
          } 2>&1 | tee svc.log
          n=$(wc -l < "$FAILS" | tr -d ' ')
          [ "$n" -eq 0 ] || { echo "::error::$n assertion(s) failed in the launchd lifecycle - see svc.log"; exit 1; }

      - name: upload logs
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: deep-test-macos
          path: |
            full-test.log
            svc.log

  docker-linux:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    # `shell: bash` is what selects `bash -eo pipefail`; without it GitHub runs
    # `bash -e`, where `cmd | tee log` reports tee's exit status and a failing
    # command inside the pipeline passes silently. Every block below pipes to tee.
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0

      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
        with:
          go-version-file: go.mod

      - name: build static binary + image (same shape goreleaser ships)
        run: |
          # dockers_v2 stages the binary at linux/<arch>/pingularity and the
          # Dockerfile COPYs $TARGETPLATFORM/pingularity, so mirror that layout here
          # (BuildKit sets TARGETPLATFORM=linux/amd64 on this amd64 runner).
          mkdir -p linux/amd64
          CGO_ENABLED=0 GOARCH=amd64 go build -o linux/amd64/pingularity .
          # `deep-test` is a purely local tag (never pushed): a version number
          # here went stale the release after it was written and made the logs
          # claim a version this build is not.
          docker build -t ghcr.io/pingular/pingularity:deep-test .

      - name: arm64 image actually runs (the release is multi-arch; only amd64 was ever executed)
        run: |
          docker run --privileged --rm tonistiigi/binfmt --install arm64
          # Stage the arm64 binary where the Dockerfile's COPY $TARGETPLATFORM/... expects it.
          mkdir -p linux/arm64
          CGO_ENABLED=0 GOARCH=arm64 go build -o linux/arm64/pingularity .
          docker buildx build --platform linux/arm64 -t ping-arm64 --load .
          # No host networking here: under QEMU we only care that the arm64
          # binary starts, opens its port and serves - the measurements it takes
          # inside an emulated container are meaningless either way.
          docker run -d --name ping-a64 -e PINGULARITY_ACCESS=network -p 9111:9000 ping-arm64
          ok=""
          for i in $(seq 1 30); do
            curl -fsS --max-time 3 http://127.0.0.1:9111/metrics >/dev/null 2>&1 && { ok=1; break; }
            sleep 2
          done
          echo "arm64 metrics -> $(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9111/metrics)"
          echo "arm64 status  -> $(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9111/api/status)"
          echo "--- arm64 container logs ---"; docker logs ping-a64 2>&1 | tail -15
          docker rm -f ping-a64
          [ -n "$ok" ] || { echo "arm64 image never became ready"; exit 1; }

      - name: image ships /var/lib/pingularity 0700 65532:65532 (exported image, BEFORE any run)
        run: |
          # The old form of this assertion read the volume AFTER the daemon ran,
          # and the daemon tightens its own container data dir at boot (store.go)
          # - so a 0755-shipping image passed anyway. Vacuous. docker create +
          # export reads what the layers say before any process ever runs.
          docker create --name perm-probe ghcr.io/pingular/pingularity:deep-test
          docker export perm-probe -o image-rootfs.tar
          docker rm perm-probe
          python3 - image-rootfs.tar <<'EOF' 2>&1 | tee -a docker.log
          import sys, tarfile
          entries = {}
          with tarfile.open(sys.argv[1]) as tf:
              for e in tf:
                  entries[e.name.lstrip("./").rstrip("/")] = e
          failed = False
          d = entries.get("var/lib/pingularity")
          if d is None or not d.isdir():
              print("/var/lib/pingularity missing from the exported image"); failed = True
          else:
              if d.mode & 0o7777 != 0o700:
                  print(f"/var/lib/pingularity mode {oct(d.mode & 0o7777)}, want 0700"); failed = True
              if (d.uid, d.gid) != (65532, 65532):
                  print(f"/var/lib/pingularity owner {d.uid}:{d.gid}, want 65532:65532"); failed = True
          if "var/lib/pingularity/.pingularity-image-dir" not in entries:
              print("volume-lineage marker .pingularity-image-dir missing"); failed = True
          if not failed:
              print("exported image: /var/lib/pingularity 0700 65532:65532 with marker - as shipped")
          sys.exit(1 if failed else 0)
          EOF

      - name: host-mode docker run (the README command)
        run: |
          # See the launchd step for why failures are counted in a file.
          FAILS=$(mktemp)
          note() { echo "ASSERT FAIL: $*"; echo 1 >> "$FAILS"; }
          code() { curl -s -o /dev/null -w '%{http_code}' --max-time 5 "$1" || echo FAIL; }
          {
            docker run -d --name ping-host --network=host --cap-add=NET_RAW \
              -v pingularity-data:/var/lib/pingularity ghcr.io/pingular/pingularity:deep-test
            for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9000/metrics >/dev/null 2>&1 && break; sleep 2; done
            m=$(code http://127.0.0.1:9000/metrics);    echo "metrics -> $m"
            st=$(code http://127.0.0.1:9000/api/status); echo "status  -> $st"
            [ "$m" = "200" ]  || note "host-mode /metrics returned $m, want 200"
            [ "$st" = "200" ] || note "host-mode /api/status returned $st, want 200"
            # The SHIPPED dir mode is proven on the exported image in the step
            # above - measuring it here, post-run, would be vacuous because the
            # daemon tightens its own dir at boot. What IS runtime behaviour is
            # the database the daemon just created: it must be 0600.
            fperm=$(docker run --rm -v pingularity-data:/d alpine stat -c '%a' /d/pingularity.db)
            echo "volume perms: db=$fperm"
            [ "$fperm" = "600" ] || note "container database is $fperm, want 600"
            docker logs ping-host 2>&1 | grep -q "group/world-accessible" \
              && note "the daemon warned about its own data directory on startup"
            echo "--- container logs ---"; docker logs ping-host 2>&1 | tail -20
            docker rm -f ping-host
          } 2>&1 | tee -a docker.log
          n=$(wc -l < "$FAILS" | tr -d ' ')
          [ "$n" -eq 0 ] || { echo "::error::$n assertion(s) failed in the host-mode run - see docker.log"; exit 1; }

      - name: compose one-shot flow (mirrors install.pingularity.dev/compose.yaml)
        run: |
          # See the launchd step for why failures are counted in a file.
          FAILS=$(mktemp)
          note() { echo "ASSERT FAIL: $*"; echo 1 >> "$FAILS"; }
          {
            # This block mirrors the canonical hosted compose (served by the dl
            # Worker at install.pingularity.dev/compose.yaml; template lives in the
            # pingularity.dev repo, dl/worker.js). Keep shapes in sync.
            mkdir -p compose-test
            cd compose-test
            # Leave nothing running if any assertion path dies mid-flow (local
            # reruns would otherwise find port 9000 taken); explicit downs still
            # run between phases.
            trap 'docker compose down --remove-orphans >/dev/null 2>&1 || true' EXIT
            cat > compose.yaml <<'EOF'
          services:
            pingularity:
              image: ghcr.io/pingular/pingularity:deep-test
              container_name: pingularity
              network_mode: host
              environment:
                - PINGULARITY_ACCESS=network
              cap_add:
                - NET_RAW
              volumes:
                - pingularity-data:/var/lib/pingularity
              restart: unless-stopped

          volumes:
            pingularity-data:
              name: pingularity-data
          EOF
            docker compose up -d
            for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9000/metrics >/dev/null 2>&1 && break; sleep 2; done
            cm=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:9000/metrics || echo FAIL)
            echo "compose host-mode metrics -> $cm"
            [ "$cm" = "200" ] || note "compose host-mode /metrics returned $cm, want 200"
            echo "--- volume carries the docker-run data (interchange check) ---"
            docker run --rm -v pingularity-data:/d alpine ls -la /d
            docker compose down
            echo "--- Docker Desktop knob variant (bridge + published port) ---"
            sed -i 's/^    network_mode: host$/    ports: ["9000:9000"]/' compose.yaml
            docker compose up -d
            for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9000/metrics >/dev/null 2>&1 && break; sleep 2; done
            bm=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:9000/metrics || echo FAIL)
            echo "compose bridge-mode metrics -> $bm"
            [ "$bm" = "200" ] || note "compose bridge-mode /metrics returned $bm, want 200"
            docker compose down
            echo "--- reverse-proxy shape: command: flags reach the Host guard (issue #16) ---"
            # PINGULARITY_OPTS is the native unit's convention; a container must get
            # flags via command:. Assert the chain on the DEFAULT image - compose
            # command: syntax, its exec-form entrypoint, the flag parser, and the
            # DNS-rebinding guard - by admitting one public Host and rejecting
            # another. The iperf variant gets the same pair in its own step below.
            sed -i 's/^    ports: \["9000:9000"\]$/    ports: ["9000:9000"]\n    command: ["-allow-host=ping.example.com"]/' compose.yaml
            docker compose up -d
            for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9000/metrics >/dev/null 2>&1 && break; sleep 2; done
            ah=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -H 'Host: ping.example.com' http://127.0.0.1:9000/ || echo FAIL)
            oh=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -H 'Host: other.example.com' http://127.0.0.1:9000/ || echo FAIL)
            echo "allow-host admitted -> $ah, other public Host -> $oh"
            [ "$ah" = "200" ] || note "command:-supplied -allow-host did not admit its Host (got $ah, want 200)"
            [ "$oh" = "403" ] || note "unlisted public Host was not rejected (got $oh, want 403)"
            args=$(docker inspect pingularity --format '{{json .Args}}')
            echo "argv: $args"
            case "$args" in *"-allow-host=ping.example.com"*) ;; *) note "compose command: flag missing from container argv";; esac
            # The variable itself must stay inert and WARNED about in a container.
            docker compose down
            # The sentinel is a value shape /etc/default/pingularity really carries
            # (-metrics-token) and must never surface in any log; the allow-host
            # part is what the inertness 403 is asserted against. Do NOT sentinel
            # on the domain - the Host guard legitimately logs rejected Hosts.
            sed -i '/^    command: \["-allow-host=ping.example.com"\]$/d' compose.yaml
            sed -i 's/^      - PINGULARITY_ACCESS=network$/      - PINGULARITY_ACCESS=network\n      - PINGULARITY_OPTS=-allow-host=ping.example.com -metrics-token=DEEP_SECRET_SENTINEL/' compose.yaml
            docker compose up -d
            for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9000/metrics >/dev/null 2>&1 && break; sleep 2; done
            eh=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -H 'Host: ping.example.com' http://127.0.0.1:9000/ || echo FAIL)
            echo "env-var variant Host -> $eh (must stay 403: nothing expands PINGULARITY_OPTS in a container)"
            [ "$eh" = "403" ] || note "PINGULARITY_OPTS env var unexpectedly took effect (got $eh, want 403)"
            # Capture logs first: a bare `docker logs | grep -q` under pipefail can
            # turn a successful early match into SIGPIPE pipeline failure.
            clogs=$(docker logs pingularity 2>&1)
            grep -Fq 'WARNING: PINGULARITY_OPTS' <<<"$clogs" || note "ignored-PINGULARITY_OPTS stderr warning missing from container logs"
            # The stderr grep above cannot see the structured replay (different
            # text, different sink) - assert the About-ring copy via /api/logs,
            # or the whole replay could be deleted with everything still green.
            # Fail CLOSED: a partial body could carry the replay phrase yet
            # truncate before leaked data, so a failed fetch is its own failure.
            if api_logs=$(curl -fsS --max-time 5 http://127.0.0.1:9000/api/logs); then
              grep -Fq 'official container images do not expand it' <<<"$api_logs" \
                || note "structured ignored-PINGULARITY_OPTS warning missing from /api/logs (About ring)"
              grep -Fq 'DEEP_SECRET_SENTINEL' <<<"$api_logs" && note "the PINGULARITY_OPTS value leaked into /api/logs"
            else
              note "fetching /api/logs failed"
            fi
            grep -Fq 'DEEP_SECRET_SENTINEL' <<<"$clogs" && note "the PINGULARITY_OPTS value leaked into container logs"
            # Keep diagnostics in docker.log without ever printing a leaked value.
            echo "--- env-leg container logs (sentinel-filtered tail) ---"
            { grep -Fv 'DEEP_SECRET_SENTINEL' <<<"$clogs" || true; } | tail -15
            docker compose down
          } 2>&1 | tee -a docker.log
          n=$(wc -l < "$FAILS" | tr -d ' ')
          [ "$n" -eq 0 ] || { echo "::error::$n assertion(s) failed in the compose flow - see docker.log"; exit 1; }

      - name: iperf3 image variant (nothing but .goreleaser.yaml ever referenced it)
        run: |
          # See the launchd step for why failures are counted in a file.
          FAILS=$(mktemp)
          note() { echo "ASSERT FAIL: $*"; echo 1 >> "$FAILS"; }
          {
            # Dockerfile.iperf ships in every release and no test ever built it, so a
            # break in it - the apt layer, the hand-recreated 65532 account debian has
            # no distroless `nonroot` for, the /data COPY that gives the volume its
            # mode - would first surface in a user's release. The amd64 binary is
            # already staged at linux/amd64/pingularity by the build step above.
            trap 'docker rm -fv ping-iperf ping-iperf-cmd ping-iperf-env ping-iperf-run iperf-srv >/dev/null 2>&1 || true; docker network rm iperf-net >/dev/null 2>&1 || true' EXIT
            docker build -f Dockerfile.iperf -t ping-iperf .
            uid=$(docker run --rm --entrypoint id ping-iperf -u)
            perm=$(docker run --rm --entrypoint stat ping-iperf -c '%a' /var/lib/pingularity)
            echo "iperf image: uid=$uid data-dir=$perm"
            [ "$uid" = "65532" ] || note "iperf image runs as uid $uid, want 65532 (what the default image uses)"
            [ "$perm" = "700" ] || note "iperf image data dir is $perm, want 700"
            docker run --rm --entrypoint iperf3 ping-iperf --version \
              || note "iperf3 does not run in the image that exists to carry it"
            docker run -d --name ping-iperf -e PINGULARITY_ACCESS=network -p 9112:9000 ping-iperf
            for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9112/metrics >/dev/null 2>&1 && break; sleep 2; done
            im=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:9112/metrics || echo FAIL)
            echo "iperf image metrics -> $im"
            [ "$im" = "200" ] || note "iperf image /metrics returned $im, want 200"
            ilogs=$(docker logs ping-iperf 2>&1)
            grep -Fq "group/world-accessible" <<<"$ilogs" \
              && note "the iperf image warned about its own data directory on startup"
            echo "--- iperf container logs ---"; tail -15 <<<"$ilogs"
            docker rm -fv ping-iperf
            echo "--- issue #16 parity: same Host-guard pair on the variant the report used ---"
            docker run -d --name ping-iperf-cmd -e PINGULARITY_ACCESS=network -p 9113:9000 ping-iperf -allow-host=ping.example.com
            for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9113/metrics >/dev/null 2>&1 && break; sleep 2; done
            iah=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -H 'Host: ping.example.com' http://127.0.0.1:9113/ || echo FAIL)
            ioh=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -H 'Host: other.example.com' http://127.0.0.1:9113/ || echo FAIL)
            iargs=$(docker inspect ping-iperf-cmd --format '{{json .Args}}')
            echo "iperf command-arg: allowed -> $iah, other -> $ioh, argv: $iargs"
            [ "$iah" = "200" ] || note "iperf image: command-arg -allow-host did not admit its Host (got $iah, want 200)"
            [ "$ioh" = "403" ] || note "iperf image: unlisted public Host not rejected (got $ioh, want 403)"
            case "$iargs" in *"-allow-host=ping.example.com"*) ;; *) note "iperf image: run-arg flag missing from container argv";; esac
            docker rm -fv ping-iperf-cmd
            docker run -d --name ping-iperf-env -p 9113:9000 -e PINGULARITY_ACCESS=network -e "PINGULARITY_OPTS=-allow-host=ping.example.com -metrics-token=DEEP_SECRET_SENTINEL" ping-iperf
            for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9113/metrics >/dev/null 2>&1 && break; sleep 2; done
            ieh=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 -H 'Host: ping.example.com' http://127.0.0.1:9113/ || echo FAIL)
            echo "iperf env-var: Host -> $ieh (must stay 403)"
            [ "$ieh" = "403" ] || note "iperf image: PINGULARITY_OPTS env var unexpectedly took effect (got $ieh, want 403)"
            elogs=$(docker logs ping-iperf-env 2>&1)
            grep -Fq 'WARNING: PINGULARITY_OPTS' <<<"$elogs" || note "iperf image: ignored-PINGULARITY_OPTS stderr warning missing from container logs"
            if iapi_logs=$(curl -fsS --max-time 5 http://127.0.0.1:9113/api/logs); then
              grep -Fq 'official container images do not expand it' <<<"$iapi_logs" \
                || note "iperf image: structured warning missing from /api/logs (About ring)"
              grep -Fq 'DEEP_SECRET_SENTINEL' <<<"$iapi_logs" && note "iperf image: the PINGULARITY_OPTS value leaked into /api/logs"
            else
              note "iperf image: fetching /api/logs failed"
            fi
            grep -Fq 'DEEP_SECRET_SENTINEL' <<<"$elogs" && note "iperf image: the PINGULARITY_OPTS value leaked into container logs"
            echo "--- iperf env-leg container logs (sentinel-filtered tail) ---"
            { grep -Fv 'DEEP_SECRET_SENTINEL' <<<"$elogs" || true; } | tail -15
            docker rm -fv ping-iperf-env
            echo "--- real iperf3 speedtest: sidecar server, engine selected via the API ---"
            # The engine needs a server, and the image itself carries iperf3, so
            # the sidecar runs from the SAME image (--entrypoint iperf3 -s): no
            # third-party image, and it doubles as proof the bundled binary works
            # in server mode. A user-defined bridge network gives the daemon
            # container DNS for the sidecar's name.
            docker network create iperf-net
            docker run -d --name iperf-srv --network iperf-net --entrypoint iperf3 ping-iperf -s
            docker run -d --name ping-iperf-run --network iperf-net -p 9114:9000 -e PINGULARITY_ACCESS=network ping-iperf
            for i in $(seq 1 15); do curl -fsS --max-time 3 http://127.0.0.1:9114/metrics >/dev/null 2>&1 && break; sleep 2; done
            # quick_setup_done releases the first-run consent hold; the rest
            # selects the engine and points it at the sidecar. The defaults
            # already run both directions plus the UDP loss/jitter pass.
            curl -fsS -X POST -H 'Content-Type: application/json' --max-time 10 \
              -d '{"quick_setup_done":true,"speed_engine":"iperf3","iperf_server":"iperf-srv:5201"}' \
              http://127.0.0.1:9114/api/settings >/dev/null || note "selecting the iperf3 engine via /api/settings failed"
            # POST /api/speedtest runs the measurement synchronously and returns
            # the stored sample (~5s per direction + the UDP pass; 180s is slack).
            sample=$(curl -fsS -X POST -H 'Content-Type: application/json' --max-time 180 http://127.0.0.1:9114/api/speedtest || true)
            echo "iperf3 sample: $sample"
            printf '%s' "$sample" > iperf-sample.json
            python3 - iperf-sample.json <<'EOF' || note "the iperf3 run did not produce a full sample (see lines above)"
          import json, sys
          try:
              with open(sys.argv[1]) as f:
                  s = json.load(f)
          except Exception as e:
              print(f"sample is not JSON: {e}"); sys.exit(1)
          bad = False
          if s.get("engine") != "iperf3":
              print(f"engine = {s.get('engine')!r}, want 'iperf3'"); bad = True
          for k in ("down_mbps", "up_mbps"):
              if not isinstance(s.get(k), (int, float)) or s.get(k) <= 0:
                  print(f"{k} = {s.get(k)!r}, want > 0"); bad = True
          for k in ("jitter_ms", "packet_loss"):
              if not isinstance(s.get(k), (int, float)):
                  print(f"{k} = {s.get(k)!r}, want a number (the UDP pass measures it)"); bad = True
          if s.get("udp_direction") not in ("down", "up"):
              print(f"udp_direction = {s.get('udp_direction')!r}, want down|up"); bad = True
          sys.exit(1 if bad else 0)
          EOF
            # ...and the sample must have LANDED, not just been returned: the
            # history API is what the dashboard and exports read.
            hist=$(curl -fsS --max-time 5 'http://127.0.0.1:9114/api/speed?mins=60' || echo '[]')
            grep -Fq '"engine":"iperf3"' <<<"$hist" || note "no iperf3 sample landed in /api/speed history"
            docker rm -fv ping-iperf-run iperf-srv
            docker network rm iperf-net
          } 2>&1 | tee -a docker.log
          n=$(wc -l < "$FAILS" | tr -d ' ')
          [ "$n" -eq 0 ] || { echo "::error::$n assertion(s) failed in the iperf3 image - see docker.log"; exit 1; }

      - name: upload logs
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: deep-test-docker
          path: docker.log