e2e test suite + bunch of bugfixes

.github/workflows/ci.yml

@@ -13,19 +13,145 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
-    runs-on: [self-hosted, x64]
+  build-and-e2e:
+    runs-on: [self-hosted, Linux, x64]
+    timeout-minutes: 90
+    # The CI box is shared - host :6650/:18080 belong to other standing stacks, so the e2e stack
+    # runs on its own ports (stack-up.sh / run-dekaf.sh honor these env vars).
+    env:
+      PULSAR_ADMIN_PORT: 28080
+      PULSAR_BROKER_PORT: 26650
+      DEKAF_PORT: 28090
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - uses: ./.github/actions/git-lfs-pull
 
+      # Codegen (proto -> ui/grpc-web + server pb) + ui build + server build + server unit tests.
       - name: Build + server tests
         shell: bash
         run: nix-shell --run 'make build'
 
       - name: UI unit tests
         shell: bash
         run: nix-shell --run 'cd ui && make test'
+
+      # No browser-install step: browsers come from the nix store (flake.nix: nixpkgs-playwright).
+      - name: Bring up Pulsar (standalone) + Dekaf server
+        shell: bash
+        run: |
+          # Kill any Dekaf leaked by a previous run - it would answer the probe with a stale binary.
+          envoy=$(lsof -t -iTCP:"$DEKAF_PORT" -sTCP:LISTEN 2>/dev/null | head -1 || true)
+          if [ -n "$envoy" ]; then
+            jvm=$(ps -o ppid= -p "$envoy" 2>/dev/null | tr -d ' ' || true)
+            sbt=""
+            if [ -n "$jvm" ] && [ "$jvm" != "1" ]; then sbt=$(ps -o ppid= -p "$jvm" 2>/dev/null | tr -d ' ' || true); fi
+            pids=$(printf '%s\n' "$envoy" "$jvm" "$sbt" | grep -vE '^1?$' | tr '\n' ' ' || true)
+            echo "Killing leftover Dekaf tree from a previous run: $pids"
+            kill $pids 2>/dev/null || true
+            sleep 3
+            kill -9 $pids 2>/dev/null || true
+          fi
+
+          nix-shell --run 'e2e/scripts/stack-up.sh'
+          nix-shell --run 'e2e/scripts/run-dekaf.sh' > "$RUNNER_TEMP/dekaf-boot.log" 2>&1 &
+
+          ready=false
+          for _ in $(seq 1 60); do
+            if curl -sf "http://localhost:$DEKAF_PORT/" >/dev/null 2>&1; then ready=true; break; fi
+            sleep 5
+          done
+          if [ "$ready" != "true" ]; then
+            echo "::error::Dekaf did not become ready on :$DEKAF_PORT within 5 minutes"
+            tail -60 "$RUNNER_TEMP/dekaf-boot.log" || true
+            exit 1
+          fi
+
+      - name: E2E - green lane
+        shell: bash
+        env:
+          DEKAF_BASE_URL: http://localhost:${{ env.DEKAF_PORT }}
+          PULSAR_ADMIN_URL: http://localhost:${{ env.PULSAR_ADMIN_PORT }}
+          PULSAR_SERVICE_URL: pulsar://localhost:${{ env.PULSAR_BROKER_PORT }}
+          TRACE: on # trace every test, not just failures - results stay navigable as artifacts
+        run: nix-shell --run 'cd e2e && sbt test'
+
+      # Always upload: per-test traces (view via `sbt "runMain harness.TraceDashboard"` or
+      # trace.playwright.dev) + JUnit XML / HTML reports.
+      # Render results on the run's Summary page; the zips stay downloadable as before.
+      - name: Test results summary
+        if: always()
+        shell: bash
+        run: |
+          python3 - >> "$GITHUB_STEP_SUMMARY" <<'EOF'
+          import glob, xml.etree.ElementTree as ET
+          tests = failures = errors = skipped = 0
+          bad = []
+          for p in sorted(glob.glob("e2e/target/test-reports/TEST-*.xml")):
+              try:
+                  root = ET.parse(p).getroot()
+              except ET.ParseError:
+                  continue
+              for s in ([root] if root.tag == "testsuite" else root.findall("testsuite")):
+                  tests += int(s.get("tests", 0)); failures += int(s.get("failures", 0))
+                  errors += int(s.get("errors", 0)); skipped += int(s.get("skipped", 0))
+                  for c in s.iter("testcase"):
+                      if c.find("failure") is not None or c.find("error") is not None:
+                          bad.append(f'{s.get("name")} :: {c.get("name")}')
+          print("## e2e results")
+          print("| tests | passed | failed | errors | skipped |")
+          print("|---|---|---|---|---|")
+          print(f"| {tests} | {tests - failures - errors - skipped} | {failures} | {errors} | {skipped} |")
+          if bad:
+              print("\n**Failing tests:**")
+              for b in bad:
+                  print(f"- `{b}`")
+          print("\n_Traces + full JUnit XML / HTML reports: run artifacts `e2e-traces`, `e2e-test-reports`._")
+          EOF
+
+      - name: Upload e2e traces (all tests)
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-traces
+          path: e2e/target/traces/*.zip
+          if-no-files-found: ignore
+
+      - name: Upload e2e test reports (JUnit XML + HTML)
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-test-reports
+          path: |
+            e2e/target/test-reports/
+            e2e/target/test-html/
+          if-no-files-found: ignore
+
+      - name: Dekaf boot log (on failure)
+        if: failure()
+        shell: bash
+        run: tail -100 "$RUNNER_TEMP/dekaf-boot.log" || true
+
+      - name: Stop the Dekaf server
+        if: always()
+        shell: bash
+        run: |
+          # Kill the envoy -> JVM -> sbt tree; a survivor would serve this binary to the next run.
+          envoy=$(lsof -t -iTCP:"$DEKAF_PORT" -sTCP:LISTEN 2>/dev/null | head -1 || true)
+          if [ -n "$envoy" ]; then
+            jvm=$(ps -o ppid= -p "$envoy" 2>/dev/null | tr -d ' ' || true)
+            sbt=""
+            if [ -n "$jvm" ] && [ "$jvm" != "1" ]; then sbt=$(ps -o ppid= -p "$jvm" 2>/dev/null | tr -d ' ' || true); fi
+            pids=$(printf '%s\n' "$envoy" "$jvm" "$sbt" | grep -vE '^1?$' | tr '\n' ' ' || true)
+            echo "Stopping Dekaf tree: $pids"
+            kill $pids 2>/dev/null || true
+            sleep 3
+            kill -9 $pids 2>/dev/null || true
+          fi
+
+      - name: Tear down the stack
+        if: always()
+        shell: bash
+        run: nix-shell --run 'e2e/scripts/stack-down.sh' || true

.github/workflows/release.yml

@@ -30,7 +30,7 @@ concurrency:
 
 jobs:
   prepare:
-    runs-on: [self-hosted, x64]
+    runs-on: [self-hosted, Linux, x64]
     outputs:
       version: ${{ steps.ver.outputs.version }}
       channel: ${{ steps.ver.outputs.channel }}
@@ -155,7 +155,7 @@ jobs:
   # quick-start compose pins, commits, tags, and pushes.
   merge:
     needs: [prepare, build]
-    runs-on: [self-hosted, x64]
+    runs-on: [self-hosted, Linux, x64]
     env:
       VER: ${{ needs.prepare.outputs.version }}
       DEKAF_IMAGE: ${{ needs.prepare.outputs.dekaf_image }}

dev-env.sh

@@ -26,6 +26,12 @@ function configure_kubectl() {
 }
 
 add_binary_dependencies_to_path
-configure_kubectl
+
+# kubectl-against-core-infra is OPTIONAL dev tooling: it needs a logged-in pulumi (or
+# PULUMI_ACCESS_TOKEN). Skip quietly when unavailable instead of printing a scary
+# "error: PULUMI_ACCESS_TOKEN must be set..." on every shell entry, locally and in CI.
+if [ -n "${PULUMI_ACCESS_TOKEN:-}" ] || pulumi whoami >/dev/null 2>&1; then
+  configure_kubectl
+fi
 
 alias k="kubectl"

e2e/.env.example

@@ -0,0 +1,12 @@
+# Load into your shell before `sbt test`:   set -a; source e2e/.env.example; set +a
+# (these `export`s reach the JVM's sys.env; a bare `X=Y` source would not.)
+# Local standalone stack (scripts/stack-up.sh) publishes admin on :18080 (host :8080 is often taken).
+export DEKAF_BASE_URL=http://localhost:8090
+export PULSAR_ADMIN_URL=http://localhost:18080
+export PULSAR_SERVICE_URL=pulsar://localhost:6650
+
+# Debug/observe knobs (uncomment):
+# export HEADED=true       # 1/true/yes/on all accepted
+# export SLOWMO=300
+# export TRACE=on          # on | off | retain-on-failure (default)
+# export PWDEBUG=1         # opens the Playwright Inspector (forces headed)