diff --git a/generator/tests/test_shared_steps.py b/generator/tests/test_shared_steps.py index 829a1236..666274fd 100644 --- a/generator/tests/test_shared_steps.py +++ b/generator/tests/test_shared_steps.py @@ -1320,3 +1320,150 @@ def test_run_issue_reconcile_refuses_a_call_with_no_row( assert "the row for this run is required" in result.stderr, result.stderr calls = Path(report_failure_env["GH_CALLS"]) assert not calls.exists(), f"reached gh before refusing: {calls.read_text()}" + + +# --------------------------------------------------------------------------- +# install-claude-binary.sh — the sandbox install, and the CDN blips it rides out +# --------------------------------------------------------------------------- + +INSTALL_CLAUDE_BINARY = REPO_ROOT / "shared" / "steps" / "install-claude-binary.sh" + +# `sudo -u USER rest…` with the -u dropped: the test has no sandbox user, and +# what is under test is the retry loop rather than the privilege drop. +FAKE_SUDO = """#!/usr/bin/env bash +[ "$1" = "-u" ] && shift 2 +exec "$@" +""" + +# Fails with the CDN's 403 for the first $CURL_FAILURES attempts, then emits an +# installer that plants a `claude` where the script looks for it. Counting in a +# file rather than a variable because each attempt is a fresh process. +FAKE_CURL = """#!/usr/bin/env bash +n=$(cat "$CURL_ATTEMPTS" 2>/dev/null || echo 0) +n=$((n + 1)) +echo "$n" > "$CURL_ATTEMPTS" +if [ "$n" -le "${CURL_FAILURES:-0}" ]; then + echo "curl: (22) The requested URL returned error: 403" >&2 + exit 22 +fi +cat <<'INSTALLER' +mkdir -p "$HOME/.local/bin" +printf '#!/usr/bin/env bash\\necho claude %s\\n' "$1" > "$HOME/.local/bin/claude" +chmod +x "$HOME/.local/bin/claude" +INSTALLER +""" + +# Records what the backoff asked for instead of waiting it out, so a test can +# assert the shape of the delay without paying it. Distinct from the +# discard-only FAKE_SLEEP above, which several fixtures share — same module, +# so reusing that name would rebind theirs. +FAKE_SLEEP_RECORDING = """#!/usr/bin/env bash +printf '%s\\n' "$1" >> "$SLEEPS" +""" + + +@pytest.fixture +def install_env(tmp_path: Path) -> dict[str, str]: + """Fake sudo/curl/sleep on PATH plus the env the install step is given.""" + bindir = _fake_bin( + tmp_path, sudo=FAKE_SUDO, curl=FAKE_CURL, sleep=FAKE_SLEEP_RECORDING + ) + agent_home = tmp_path / "agent-home" + agent_home.mkdir() + + return { + "PATH": f"{bindir}:/usr/bin:/bin", + "SANDBOX": "tend-sandbox", + "AGENT_HOME": str(agent_home), + "CLAUDE_VERSION": "2.1.220", + "CURL_ATTEMPTS": str(tmp_path / "curl-attempts"), + "SLEEPS": str(tmp_path / "sleeps"), + } + + +def _run_install(env: dict[str, str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", str(INSTALL_CLAUDE_BINARY)], + env=env, + capture_output=True, + text=True, + ) + + +def _attempts(env: dict[str, str]) -> int: + return int(Path(env["CURL_ATTEMPTS"]).read_text().strip()) + + +def _sleeps(env: dict[str, str]) -> list[int]: + path = Path(env["SLEEPS"]) + if not path.exists(): + return [] + return [int(line) for line in path.read_text().split()] + + +def test_install_claude_binary_rides_out_a_403_burst( + install_env: dict[str, str], +) -> None: + """Four straight 403s must not cost the run. + + Observed in production: every attempt inside a ~15s window answered 403 + while sibling matrix legs installed fine seconds either side. Three + attempts is too few to cross a blip that short, and the step failing this + early leaves no outage row, so the run vanishes without a trace. + """ + install_env["CURL_FAILURES"] = "4" + + result = _run_install(install_env) + + assert result.returncode == 0, ( + f"gave up on a blip it should have ridden out; stderr:\n{result.stderr}" + ) + assert _attempts(install_env) == 5, _attempts(install_env) + assert (Path(install_env["AGENT_HOME"]) / ".local/bin/claude").exists() + + +def test_install_claude_binary_backs_off_exponentially( + install_env: dict[str, str], +) -> None: + """Each wait at least doubles, so the window spans a blip rather than a burst. + + A flat delay spends every attempt inside the first few seconds, which is + the failure above. Only the floors are pinned — the jitter above each one + is deliberate and free to vary. + """ + install_env["CURL_FAILURES"] = "99" + + _run_install(install_env) + + assert _sleeps(install_env) == pytest.approx([5, 10, 20, 40], abs=9), ( + f"backoff did not double: {_sleeps(install_env)}" + ) + assert all( + actual >= floor + for actual, floor in zip(_sleeps(install_env), [5, 10, 20, 40], strict=True) + ), f"slept less than the backoff floor: {_sleeps(install_env)}" + + +def test_install_claude_binary_reddens_when_every_attempt_fails( + install_env: dict[str, str], +) -> None: + """A CDN that stays down still fails the step, and says how hard it tried.""" + install_env["CURL_FAILURES"] = "99" + + result = _run_install(install_env) + + assert result.returncode != 0, result.stdout + assert "after 5 attempts" in result.stdout, result.stdout + assert _attempts(install_env) == 5, _attempts(install_env) + + +def test_install_claude_binary_installs_first_try_without_sleeping( + install_env: dict[str, str], +) -> None: + """The happy path is one fetch and no delay — the retry must not cost it.""" + result = _run_install(install_env) + + assert result.returncode == 0, result.stderr + assert _attempts(install_env) == 1, _attempts(install_env) + assert _sleeps(install_env) == [], _sleeps(install_env) + assert "claude 2.1.220" in result.stdout, result.stdout diff --git a/shared/steps/install-claude-binary.sh b/shared/steps/install-claude-binary.sh index 72096777..eb34ba01 100755 --- a/shared/steps/install-claude-binary.sh +++ b/shared/steps/install-claude-binary.sh @@ -33,17 +33,30 @@ set -euo pipefail # `set -o pipefail` is required: without it a curl failure passes empty # stdin to the downstream `bash -s --`, which exits 0, masking the # failure so the loop breaks after one attempt without retrying. -for i in 1 2 3; do +# +# Five attempts backing off exponentially, not three at a flat 5s. The old +# window spent all three attempts inside ~15s, which is short enough for one +# CDN blip to take the whole run: the step goes red, and because `Report +# failure` is gated on the agent step, a failure this early leaves no outage +# row either, so the run disappears silently. +# +# Jittered, because the legs of a matrix workflow install concurrently from +# one runner's egress address. A rate limit hits them together, and an +# unjittered backoff has them retry together too — every leg reproducing the +# burst that tripped it. The jitter spreads the retries out. +ATTEMPTS=5 +for i in $(seq 1 "$ATTEMPTS"); do if timeout 60 bash -c "set -o pipefail; \ curl -fsSL https://claude.ai/install.sh | bash -s -- '$CLAUDE_VERSION'"; then break fi - if [ "$i" = 3 ]; then - echo "::error::failed to install claude $CLAUDE_VERSION after 3 attempts" + if [ "$i" -eq "$ATTEMPTS" ]; then + echo "::error::failed to install claude $CLAUDE_VERSION after $ATTEMPTS attempts" exit 1 fi - echo "Install attempt $i failed; retrying" - sleep $((i * 5)) + BACKOFF=$((5 * 2 ** (i - 1) + RANDOM % 10)) + echo "Install attempt $i failed; retrying in ${BACKOFF}s" + sleep "$BACKOFF" done EOF if ! sudo -u "$SANDBOX" test -x "$AGENT_HOME/.local/bin/claude"; then