From be8cd1ddce3284402f072d99967d23ae8c1315a8 Mon Sep 17 00:00:00 2001 From: Andre Levesque <0sniffs_scaled@icloud.com> Date: Sat, 18 Jul 2026 07:39:00 -0400 Subject: [PATCH] ci: retry E2E wrangler boot once with port cleanup (#625) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e-env server-start step failed twice in two days with "Server failed to start after 180 seconds", cascading to full-suite red — but CI logs show no attempt was actually a slow boot (worst was seconds), so the cause is a transient boot hiccup or a leftover process holding :8788 from a prior step, which more waiting can't fix. Refactor the start into functions and retry the BOOT ONCE (never test failures, which are a separate step): on attempt-1 failure, kill the wrangler process AND any pid still bound to :8788 (wrangler forks esbuild + workerd children that outlive the parent), then relaunch once. A server that truly never starts still fails the job. Budget stays 180s (padding wouldn't help this failure mode); logs surface every 20s. Mirrored into the Makefile e2e-serve target to keep local == CI. Closes #625 Co-Authored-By: Claude Fable 5 --- .github/actions/e2e-env/action.yml | 98 +++++++++++++++++++++--------- Makefile | 24 ++++++-- 2 files changed, 89 insertions(+), 33 deletions(-) diff --git a/.github/actions/e2e-env/action.yml b/.github/actions/e2e-env/action.yml index 7c2bc6c6..4f26e583 100644 --- a/.github/actions/e2e-env/action.yml +++ b/.github/actions/e2e-env/action.yml @@ -87,34 +87,76 @@ runs: echo "✓ Database initialization complete" - name: Start Pages Functions server + # Retries the SERVER BOOT ONLY, once, on a fresh process — never on test + # failures (those are a separate step). A dead process or an unmet 180s + # budget on attempt 1 kills any leftover wrangler/workerd/esbuild holding + # :8788 and relaunches once before failing the job for real. See #625: + # no observed CI failure has actually been a slow boot (worst seen was a + # few seconds), so the budget is left at 180s rather than padded on + # speculation — the retry covers one-off boot hiccups a longer wait + # wouldn't fix anyway (e.g. a stuck port from a prior process). shell: bash run: | - ./frontend/node_modules/.bin/wrangler pages dev frontend/dist --port 8788 --persist-to .wrangler/state > /tmp/wrangler.log 2>&1 & - WRANGLER_PID=$! - - echo "Waiting for server to start..." - i=1 - while [ $i -le 90 ]; do - if ! kill -0 $WRANGLER_PID 2>/dev/null; then - echo "✗ Wrangler process exited unexpectedly" - echo "Wrangler logs:" - cat /tmp/wrangler.log - exit 1 - fi - if curl -s http://localhost:8788/ > /dev/null 2>&1; then - echo "✓ Server is ready on port 8788!" - exit 0 - fi - if [ $((i % 20)) -eq 0 ]; then - echo "--- Wrangler log (iteration $i) ---" - tail -20 /tmp/wrangler.log 2>/dev/null - echo "---" - fi - echo "Waiting for server... ($i/90)" - sleep 2 - i=$((i + 1)) - done - echo "✗ Server failed to start after 180 seconds" - echo "Wrangler logs:" - cat /tmp/wrangler.log + MAX_ITER=90 # 90 * 2s = 180s budget per attempt + LOG_EVERY=10 # surface wrangler.log every 10 iterations (20s) instead of 40s + + start_wrangler() { + ./frontend/node_modules/.bin/wrangler pages dev frontend/dist --port 8788 --persist-to .wrangler/state > /tmp/wrangler.log 2>&1 & + WRANGLER_PID=$! + } + + kill_wrangler() { + kill "$WRANGLER_PID" 2>/dev/null || true + # wrangler forks esbuild + workerd children that don't die with the + # parent; free the port explicitly so the retry doesn't fail to bind. + for pid in $(lsof -ti:8788 2>/dev/null); do + kill -9 "$pid" 2>/dev/null || true + done + } + + wait_for_server() { + attempt=$1 + echo "Waiting for server to start (attempt $attempt/2)..." + i=1 + while [ $i -le $MAX_ITER ]; do + if ! kill -0 $WRANGLER_PID 2>/dev/null; then + echo "✗ Wrangler process exited unexpectedly (attempt $attempt)" + echo "Wrangler logs:" + cat /tmp/wrangler.log + return 1 + fi + if curl -s http://localhost:8788/ > /dev/null 2>&1; then + echo "✓ Server is ready on port 8788! (attempt $attempt, $((i * 2))s)" + return 0 + fi + if [ $((i % LOG_EVERY)) -eq 0 ]; then + echo "--- Wrangler log so far (attempt $attempt, iteration $i) ---" + tail -20 /tmp/wrangler.log 2>/dev/null + echo "---" + fi + echo "Waiting for server... ($i/$MAX_ITER, attempt $attempt)" + sleep 2 + i=$((i + 1)) + done + echo "✗ Server failed to start after $((MAX_ITER * 2)) seconds (attempt $attempt)" + echo "Wrangler logs:" + cat /tmp/wrangler.log + return 1 + } + + start_wrangler + if wait_for_server 1; then + exit 0 + fi + + echo "Startup attempt 1 failed — killing any leftover process and retrying boot once..." + kill_wrangler + sleep 2 + + start_wrangler + if wait_for_server 2; then + exit 0 + fi + + echo "✗ Server failed to start after 2 attempts — genuine startup failure, not transient boot slowness" exit 1 diff --git a/Makefile b/Makefile index 3a491bc7..4816902c 100644 --- a/Makefile +++ b/Makefile @@ -61,11 +61,25 @@ e2e-setup: build ## Init + seed an isolated local D1 for E2E (safe to re-run) SEED_SQL=$$(node scripts/seed-e2e-admin.mjs --email "$(E2E_ADMIN_EMAIL)" --password "$(E2E_ADMIN_PASSWORD)"); \ $(WRANGLER) d1 execute settimes-production-db --local --persist-to $(E2E_STATE) --command="$$SEED_SQL" -e2e-serve: ## Start wrangler for E2E in the background (writes pidfile) - $(WRANGLER) pages dev frontend/dist --port 8788 --persist-to $(E2E_STATE) > $(E2E_STATE)/wrangler.log 2>&1 & \ - echo $$! > $(E2E_PID); \ - i=0; until curl -s -o /dev/null http://localhost:8788/ || [ $$i -ge 45 ]; do i=$$((i+1)); sleep 2; done; \ - curl -s -o /dev/null http://localhost:8788/ || { echo "wrangler failed to start; log:"; tail -20 $(E2E_STATE)/wrangler.log; exit 1; } +e2e-serve: ## Start wrangler for E2E in the background (writes pidfile); retries boot once, never test failures (mirrors e2e-env action.yml, #625) + try_boot() { \ + $(WRANGLER) pages dev frontend/dist --port 8788 --persist-to $(E2E_STATE) > $(E2E_STATE)/wrangler.log 2>&1 & \ + echo $$! > $(E2E_PID); \ + i=1; \ + while [ $$i -le 90 ]; do \ + kill -0 "$$(cat $(E2E_PID))" 2>/dev/null || { echo "wrangler process exited unexpectedly (attempt $$1)"; tail -20 $(E2E_STATE)/wrangler.log; return 1; }; \ + curl -s -o /dev/null http://localhost:8788/ && { echo "server ready on port 8788 (attempt $$1, $$((i*2))s)"; return 0; }; \ + [ $$((i % 10)) -eq 0 ] && { echo "--- wrangler log so far (attempt $$1, iteration $$i) ---"; tail -20 $(E2E_STATE)/wrangler.log; }; \ + i=$$((i+1)); sleep 2; \ + done; \ + echo "wrangler failed to start after 180s (attempt $$1)"; tail -20 $(E2E_STATE)/wrangler.log; return 1; \ + }; \ + try_boot 1 && exit 0; \ + kill "$$(cat $(E2E_PID))" 2>/dev/null || true; \ + for pid in $$(lsof -ti:8788 2>/dev/null); do kill -9 "$$pid" 2>/dev/null || true; done; \ + sleep 2; \ + try_boot 2 && exit 0; \ + echo "wrangler failed to start after 2 attempts"; exit 1 e2e-run: ## Run Playwright (server must be up; credentials required even for --list) ADMIN_EMAIL="$(E2E_ADMIN_EMAIL)" ADMIN_PASSWORD="$(E2E_ADMIN_PASSWORD)" npx playwright test $(SPEC) --reporter=list