From 31f7bc79bf98a62571f9012b3a2ee33e833da3bf Mon Sep 17 00:00:00 2001 From: swackhamer Date: Tue, 4 Aug 2026 21:51:47 -0500 Subject: [PATCH] fix(fleet): create the staging worktree before the recut can use it poll_once ran its staleness re-cut check BEFORE ensure_staging_worktree. recut_squad_branch's first act is checkout_detached(staging_path, ...), which hands staging_path to subprocess as cwd -- so on a host where ~/.oxidex/worktrees/squad-staging/ does not exist yet (a fresh machine, or after stop_parallel_fix.py reaped the worktrees) the recut raised FileNotFoundError before the one function that would have CREATED that worktree ever ran. Measured 2026-08-04: all 14 mergers died within one second of launch. Every squad/ branch was stale against a main that had moved 249 commits, so should_recut fired for all of them simultaneously and the whole merger tier went down together -- the failure mode where workers keep producing fixes with no publish path at all. ensure_staging_worktree is idempotent (reuse-in-place on an existing directory), so hoisting it above the recut check costs one cheap reset/checkout and makes the recut path survive a cold start. Every pre-existing poll_once test passes check_recut=False, which is why this ordering survived: the crashing path was never exercised. The new test drives poll_once with check_recut=True against an absent staging directory and reproduces the exact production traceback. Co-Authored-By: Claude Opus 5 --- scripts/squad_merge_loop.py | 16 ++++++++++-- scripts/test_squad_merge_loop.py | 44 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/scripts/squad_merge_loop.py b/scripts/squad_merge_loop.py index 8cf1423f6..452c70b7a 100644 --- a/scripts/squad_merge_loop.py +++ b/scripts/squad_merge_loop.py @@ -1559,6 +1559,20 @@ def poll_once(*, repo_root, squad, home, staging_dir, squads_toml_path=DEFAULT_S """ heartbeat_fn = heartbeat_fn or (lambda: None) squad_branch_ref = staging_branch(squad) + + # BEFORE the recut check, not after: recut_squad_branch's very first act is + # checkout_detached(staging_path, ...), which hands staging_path to + # subprocess as cwd. If the staging worktree does not exist yet -- a fresh + # host, or after stop_parallel_fix.py reaped the worktrees -- that raises + # FileNotFoundError from inside the recut, killing the merger before the + # one function that would have CREATED the worktree ever ran. Measured + # 2026-08-04: all 14 mergers died within a second of launch, because every + # squad branch was stale against a main that had moved 249 commits, so the + # recut clause fired for all of them simultaneously. ensure_staging_worktree + # is idempotent (reuse-in-place on an existing dir), so hoisting it costs a + # cheap reset/checkout and makes the recut path survive a cold start. + ensure_staging_worktree(repo_root, staging_dir, squad, origin_ref=origin_ref, log_fn=log_fn) + if check_recut and should_recut(repo_root, squad_branch_ref, origin_ref, recut_staleness_seconds, now_fn): log_fn(f"[{squad}] squad/{squad} is stale vs {origin_ref} -- re-cutting before this poll") @@ -1569,8 +1583,6 @@ def poll_once(*, repo_root, squad, home, staging_dir, squads_toml_path=DEFAULT_S else: recut_result = None - ensure_staging_worktree(repo_root, staging_dir, squad, origin_ref=origin_ref, log_fn=log_fn) - status_path = squad_status_file(home, squad) quarantine_entries = load_quarantine(quarantine_ledger_path(home)) # Every squad that could be racing this one for a file. Read once per diff --git a/scripts/test_squad_merge_loop.py b/scripts/test_squad_merge_loop.py index 22fe38c93..0c97fcf82 100644 --- a/scripts/test_squad_merge_loop.py +++ b/scripts/test_squad_merge_loop.py @@ -1230,6 +1230,50 @@ def comparison_fn(staging, cache, fmt, suffix): ) +class PollOnceMissingStagingWorktreeTests(GitRepoTestCase): + """The recut path must not assume the staging worktree already exists. + + poll_once runs its staleness check BEFORE ensure_staging_worktree, and + recut_squad_branch's first act is checkout_detached(staging_path, ...), + which passes staging_path as subprocess cwd. On a host where + ~/.oxidex/worktrees/squad-staging/ is absent -- a fresh machine, + or after stop_parallel_fix.py reaped the worktrees -- that raises + FileNotFoundError before the function that would have CREATED it ever + runs. Measured 2026-08-04: all 14 mergers died within one second of + launch for exactly this reason, because every squad branch was stale + against a main that had moved 249 commits, so the recut clause fired + for all of them at once. + + Every other poll_once test passes check_recut=False, which is why this + ordering survived: the crashing path was never exercised. + """ + + def _squads_toml(self): + path = self.tmp / "squads.toml" + path.write_text('[squads.nikon]\nformats = ["NEF"]\n') + return path + + def test_recut_creates_the_staging_worktree_instead_of_crashing(self): + repo = self.make_repo() + git(repo, "branch", "squad/nikon") + base_sha = git_out(repo, "rev-parse", "main").strip() + commit_ts = int(git_out(repo, "log", "-1", "--format=%ct", base_sha).strip()) + home = self.tmp / "home" + staging = self.tmp / "staging-nikon" + self.assertFalse(staging.exists(), "precondition: staging worktree is absent") + + sml.poll_once( + repo_root=repo, squad="nikon", home=home, staging_dir=staging, + squads_toml_path=self._squads_toml(), cache_dir="/unused", origin_ref="main", + batch_commits=10, batch_seconds=900, + now_fn=lambda: commit_ts + 10_000, + recut_staleness_seconds=10, check_recut=True, + log_fn=lambda *a: None, + ) + + self.assertTrue(staging.is_dir(), "poll_once must have created the staging worktree") + + class PollOnceSquadSlotBranchIntegrationTests(GitRepoTestCase): """Squad-mode dispatch (spec S2) creates model-fix-parallel-- branches, not the legacy model-fix-parallel- naming