From 6c8fbb76f480d339ae0d059459e181f69a6f24f8 Mon Sep 17 00:00:00 2001 From: "C. Colby Thomson" Date: Wed, 15 Jul 2026 02:26:41 -0700 Subject: [PATCH] env update: defer hardware smoke to PR review; fix gh repo + manifest note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The weekly env-update cron runs on the jumpbox (Alpine musl, no GPU). Its smoke step (`uv run --project envs/

python envs/smoke.py --gpu`, run locally — `forge run` only leases the remote host) could never pass there: the CUDA envs can't even build (bitsandbytes/torch ship glibc-only wheels) and there is no GPU to matmul on. Every run recorded ok:false on a smoke failure that was environmental, not a real regression. Since the PR is never auto-merged, defer hardware smoke to the reviewer: - cmd_update no longer runs smoke; marks it deferred with the smoke_host. - report `ok` now means "all locks re-resolved without error" — exactly what the jumpbox can attest. - _pr_body embeds the exact per-profile smoke command to run on the smoke_host before merging. - gh pr create gets `-R GoodAncestor/forge` (origin uses a github-forge SSH alias gh can't map to a repo on its own) — the other half of the pr_error. - drop the now-meaningless --no-smoke flag. Also: correct the stale gutenberg-mirror manifest note (~60GB -> >150GB and growing; size_bytes stays null as an unmeasurable rsync tree). --- datasets/manifest.yaml | 5 +++-- src/forge/envupdate.py | 50 ++++++++++++++++++++++++++--------------- tests/test_envupdate.py | 31 ++++++++++++++++++++++--- 3 files changed, 63 insertions(+), 23 deletions(-) diff --git a/datasets/manifest.yaml b/datasets/manifest.yaml index c9ecf79..eecfcc1 100644 --- a/datasets/manifest.yaml +++ b/datasets/manifest.yaml @@ -109,8 +109,9 @@ datasets: approved: true size_bytes: null nas_path: gutenberg/ - notes: Full mirror ~60GB via rsync; post-1930 PD slice filtered downstream. Fetched - with rsync, not HTTP. + notes: Full aleph mirror is large (>150GB and growing — all formats/mirrors, not + the old ~60GB estimate); post-1930 PD slice filtered downstream. Fetched with + rsync, not HTTP. size_bytes stays null (rsync tree, unmeasurable up front). - name: chronicling-america-1930-1963 source: url://https://chroniclingamerica.loc.gov/ tier: historical-1930-1980 diff --git a/src/forge/envupdate.py b/src/forge/envupdate.py index 719663b..700ea58 100644 --- a/src/forge/envupdate.py +++ b/src/forge/envupdate.py @@ -47,9 +47,10 @@ def write_report(diffs, smoke, report_dir): report_dir = Path(report_dir) report_dir.mkdir(parents=True, exist_ok=True) stamp = datetime.now(timezone.utc).strftime("%Y-%m-%d") - ok = (not any("error" in d for d in diffs.values()) - and all(s.get("outcome") in ("completed", "skipped") - for s in smoke.values())) + # ok reflects only what this host can attest: that every profile's lock + # re-resolved without error. Hardware smoke is deferred to PR review (see + # cmd_update), so it is advisory in the report, not a gate here. + ok = not any("error" in d for d in diffs.values()) payload = {"generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "diffs": diffs, "smoke": smoke, "ok": ok} out = report_dir / f"env-update-{stamp}.json" @@ -60,7 +61,11 @@ def write_report(diffs, smoke, report_dir): def _pr_body(diffs, smoke): - lines = ["Automated env lockfile update.", ""] + lines = ["Automated env lockfile update.", "", + "⚠ Hardware smoke is NOT run by this job — it runs on the GPU-less " + "jumpbox, where these CUDA envs can't build (glibc-only wheels) and " + "there is no GPU. Run each changed profile's smoke command below on " + "its smoke_host before merging.", ""] for prof, d in diffs.items(): lines.append(f"## {prof}") if "error" in d: @@ -72,7 +77,13 @@ def _pr_body(diffs, smoke): lines.append(f"- {name}: added {v}") for name, v in sorted(d["removed"].items()): lines.append(f"- {name}: removed (was {v})") - lines.append(f"- smoke: {smoke.get(prof, {}).get('outcome', 'not run')}") + host = smoke.get(prof, {}).get("smoke_host") + if host: + lines.append(f"- smoke ({smoke[prof]['outcome']}): run on `{host}` " + f"before merge — `uv run forge --json run " + f"recipes/smoke.yaml --host {host} --profile {prof}`") + else: + lines.append(f"- smoke: no smoke_host configured for {prof}") lines.append("") return "\n".join(lines) @@ -86,15 +97,15 @@ def cmd_update(args): continue diffs[name] = update_profile(d) changed = any(diffs[name].get(k) for k in ("changed", "added", "removed")) - if args.no_smoke or not changed or not p.get("smoke_host"): - smoke[name] = {"outcome": "skipped"} - continue - r = subprocess.run( - ["uv", "run", "forge", "--json", "run", "recipes/smoke.yaml", - "--host", p["smoke_host"], "--profile", name], - cwd=str(REPO_ROOT), capture_output=True, text=True) - smoke[name] = {"outcome": "completed" if r.returncode == 0 else "failed", - "exit": r.returncode} + # Hardware smoke is deliberately NOT run here. This job runs on the + # jumpbox (Alpine musl, no GPU): the CUDA envs can't even build there + # (bitsandbytes/torch ship glibc-only wheels) and there is no GPU to + # matmul on. The PR is never auto-merged, so smoke is deferred to the + # reviewer, who runs it on the profile's smoke_host before merging — + # the exact command is embedded in the PR body (see _pr_body). + smoke[name] = {"outcome": "deferred" if (changed and p.get("smoke_host")) + else "skipped", + "smoke_host": p.get("smoke_host")} report = write_report(diffs, smoke, args.report_dir) any_change = any(d.get(k) for d in diffs.values() for k in ("changed", "added", "removed")) @@ -113,8 +124,11 @@ def cmd_update(args): cwd=REPO_ROOT, check=True) subprocess.run(["git", "push", "-u", "origin", branch], cwd=REPO_ROOT, check=True) - r = subprocess.run(["gh", "pr", "create", "--title", - f"env lock update {stamp}", "--body", + # -R is required: origin uses a `github-forge` SSH host alias + # (deploy key), which gh can't map back to a GitHub repo on its + # own, so it can't infer the target repo from the remote. + r = subprocess.run(["gh", "pr", "create", "-R", "GoodAncestor/forge", + "--title", f"env lock update {stamp}", "--body", _pr_body(diffs, smoke)], cwd=REPO_ROOT, capture_output=True, text=True, check=True) pr_url = r.stdout.strip() @@ -135,8 +149,8 @@ def cmd_update(args): def register_update(esub): - up = esub.add_parser("update", help="re-resolve lockfiles, smoke, open PR") - up.add_argument("--no-smoke", action="store_true") + up = esub.add_parser( + "update", help="re-resolve lockfiles and open PR (smoke deferred to review)") up.add_argument("--no-pr", action="store_true") up.add_argument("--report-dir", default=str(REPO_ROOT / "reports")) up.set_defaults(func=cmd_update) diff --git a/tests/test_envupdate.py b/tests/test_envupdate.py index ec078d7..2eb4429 100644 --- a/tests/test_envupdate.py +++ b/tests/test_envupdate.py @@ -128,14 +128,39 @@ def fake_run(cmd, cwd=None, check=False, capture_output=False, text=False): report_data = json.loads(Path(payload["report"]).read_text()) assert report_data["ok"] is True + # smoke is deferred to PR review, not run by this job + assert report_data["smoke"]["cuda-x"]["outcome"] == "deferred" + assert report_data["smoke"]["cuda-x"]["smoke_host"] == "h1" add_calls = [c for c in calls if c[:2] == ["git", "add"]] assert add_calls == [["git", "add", "envs/cuda-x/uv.lock"]] assert calls[-1] == ["git", "checkout", "main"] - smoke_calls = [c for c in calls if c[:2] == ["uv", "run"]] - assert smoke_calls == [["uv", "run", "forge", "--json", "run", "recipes/smoke.yaml", - "--host", "h1", "--profile", "cuda-x"]] + # no hardware smoke subprocess runs anymore (no `uv run` calls at all) + assert [c for c in calls if c[:2] == ["uv", "run"]] == [] + # gh must target the repo explicitly (origin uses a github-forge SSH alias) + gh_calls = [c for c in calls if c[:1] == ["gh"]] + assert gh_calls and gh_calls[0][:5] == [ + "gh", "pr", "create", "-R", "GoodAncestor/forge"] + + +def test_pr_body_defers_smoke_to_host(): + diffs = {"cuda-x": {"changed": {"torch": ["2.11.0", "2.10.0"]}, + "added": {}, "removed": {}}} + smoke = {"cuda-x": {"outcome": "deferred", "smoke_host": "alien03"}} + body = envupdate._pr_body(diffs, smoke) + assert "NOT run" in body + assert "torch: 2.11.0 -> 2.10.0" in body + # reviewer gets the exact hardware-smoke command for the profile's host + assert "--host alien03 --profile cuda-x" in body + + +def test_pr_body_no_smoke_host(): + diffs = {"cuda-dc": {"changed": {"torch": ["2.11.0", "2.10.0"]}, + "added": {}, "removed": {}}} + smoke = {"cuda-dc": {"outcome": "skipped", "smoke_host": None}} + body = envupdate._pr_body(diffs, smoke) + assert "no smoke_host configured for cuda-dc" in body def test_cmd_update_pr_failure_recovers_branch(tmp_path, monkeypatch, capsys):