From 60081990a64175a3183cd3562556502ba29214f2 Mon Sep 17 00:00:00 2001 From: ProtocolWarden Date: Wed, 15 Jul 2026 00:33:22 -0400 Subject: [PATCH] fix: OperationsCenter stop Rule 8 retries on backend-capacity planner failures --- .../entrypoints/board_worker/outcomes.py | 17 +++++++++++++++++ .../entrypoints/maintenance/board_unblock.py | 5 +++++ .../board_worker/test_outcomes_cov.py | 17 +++++++++++++++++ .../maintenance/test_board_unblock.py | 13 +++++++++++++ .../maintenance/test_board_unblock_cov.py | 14 ++++++++++++++ 5 files changed, 66 insertions(+) diff --git a/src/operations_center/entrypoints/board_worker/outcomes.py b/src/operations_center/entrypoints/board_worker/outcomes.py index ae3fbc0e..4ae843d8 100644 --- a/src/operations_center/entrypoints/board_worker/outcomes.py +++ b/src/operations_center/entrypoints/board_worker/outcomes.py @@ -40,6 +40,11 @@ # fault, so we never terminate on unknown). See CODE_FAILURE_RETRY_CAP.md. _CODE_FAILURE_CATEGORIES = frozenset({"validation_failed", "no_changes"}) _TERMINAL_STATE_NAMES = {"cancelled", "done"} +_BACKEND_CAPACITY_REASON_LABEL = "blocked-reason: backend-capacity" +_BACKEND_CAPACITY_REASON_SNIPPETS = ( + "stage planner received non-json from agent", + "session limit or error", +) # ── Atomic failure transition ───────────────────────────────────────────────── @@ -97,6 +102,13 @@ def read_improve_output(workspace: Path) -> list[dict]: return [item for item in raw[:5] if isinstance(item, dict) and item.get("title")] +def _is_backend_capacity_failure(category: str, reason: str) -> bool: + if category != "backend_error": + return False + lowered = reason.lower() + return all(snippet in lowered for snippet in _BACKEND_CAPACITY_REASON_SNIPPETS) + + # ── Success handler ─────────────────────────────────────────────────────────── @@ -510,6 +522,11 @@ def handle_failure( # 26 policy-blocked re-dispatches in one hour). This label marks the block # as policy-driven so Rule 8 can exclude it. add_label(client, issue, "blocked-reason: policy") + elif _is_backend_capacity_failure(category, reason): + # Fallback dispatch hit a backend-capacity/error signature and returned + # no structured planner output; immediate clean-retry recycling just + # burns the next backend slot with the same non-JSON failure. + add_label(client, issue, _BACKEND_CAPACITY_REASON_LABEL) if executor_signal: add_label(client, issue, f"executor-signal: {executor_signal}") if "sigkill" in executor_signal.lower(): diff --git a/src/operations_center/entrypoints/maintenance/board_unblock.py b/src/operations_center/entrypoints/maintenance/board_unblock.py index dc251012..34948c8f 100644 --- a/src/operations_center/entrypoints/maintenance/board_unblock.py +++ b/src/operations_center/entrypoints/maintenance/board_unblock.py @@ -79,6 +79,9 @@ - Not "blocked-reason: policy" (deterministic policy gate — e.g. review.required — that will re-block identically on every retry; recycling it Blocked->Backlog-> Ready for AI is a closed loop, not a transient infra failure) + - Not "blocked-reason: backend-capacity" (planner/backend returned no structured + output under a backend-capacity condition; immediate recycle just repeats the + same non-JSON/session-limit failure on the next backend slot) - Blocked for at least --clean-blocked-min-minutes (default 5) minutes → move to Backlog for retry. These represent pre-execution failures (workspace preparation errors, missing @@ -268,6 +271,7 @@ def _dispatch_cooldown_reason( _HANDOFF_IMPROVEMENT_LABEL = "handoff-reason: improvement_applied" _PR_URL_PREFIX = "pr-url:" _BLOCKED_REASON_POLICY_LABEL = "blocked-reason: policy" +_BLOCKED_REASON_BACKEND_CAPACITY_LABEL = "blocked-reason: backend-capacity" _OPEN_PR_GATE_LABEL = "OPEN_PR_GATE" @@ -803,6 +807,7 @@ def _apply_rules( and not _has_label_prefix(labels, "executor-exit-code:") and not _has_label_prefix(labels, _BLOCKED_BY_PREFIX) and not _has_label(labels, _BLOCKED_REASON_POLICY_LABEL) + and not _has_label(labels, _BLOCKED_REASON_BACKEND_CAPACITY_LABEL) ) if is_clean_blocked: updated_at = _parse_updated_at(issue) diff --git a/tests/unit/entrypoints/board_worker/test_outcomes_cov.py b/tests/unit/entrypoints/board_worker/test_outcomes_cov.py index b7bf47b3..e2026e4e 100644 --- a/tests/unit/entrypoints/board_worker/test_outcomes_cov.py +++ b/tests/unit/entrypoints/board_worker/test_outcomes_cov.py @@ -711,6 +711,23 @@ def test_handle_failure_non_policy_category_no_policy_label(): assert "blocked-reason: policy" not in _all_update_labels(client) +def test_handle_failure_backend_capacity_adds_structured_label(): + client = _make_client() + issue = {"id": "b1", "labels": []} + outcomes.handle_failure( + client, + issue, + "goal", + "goal", + { + "failure_category": "backend_error", + "failure_reason": "Stage planner received non-JSON from agent (session limit or error): ", + }, + _make_settings(), + ) + assert "blocked-reason: backend-capacity" in _all_update_labels(client) + + def test_handle_failure_transient_category_does_not_count(): # Env/transient categories are NOT code failures — must not bump the counter. for cat in ("backend_error", "timeout", "unknown", "scope_too_wide"): diff --git a/tests/unit/entrypoints/maintenance/test_board_unblock.py b/tests/unit/entrypoints/maintenance/test_board_unblock.py index ff737ca4..0fd60e1c 100644 --- a/tests/unit/entrypoints/maintenance/test_board_unblock.py +++ b/tests/unit/entrypoints/maintenance/test_board_unblock.py @@ -80,6 +80,19 @@ def test_rule8_skip_spec_author_too_young(): assert len(retry_actions) == 0 +def test_rule8_skip_backend_capacity_blocked(): + """Rule 8 must NOT re-queue backend-capacity failures as clean retries.""" + issue = _issue( + "t3b", + state="Blocked", + labels=["task-kind: spec-author", "blocked-reason: backend-capacity"], + updated_at="2026-05-28T10:00:00+00:00", + ) + actions = _apply_rules([issue], **_RULES_KWARGS) + retry_actions = [a for a in actions if a["rule"] == "CLEAN_BLOCKED_RETRY"] + assert len(retry_actions) == 0 + + # --- Rule 9: SPEC_AUTHOR_BACKLOG_PROMOTE --- diff --git a/tests/unit/entrypoints/maintenance/test_board_unblock_cov.py b/tests/unit/entrypoints/maintenance/test_board_unblock_cov.py index 23807166..ce4ff381 100644 --- a/tests/unit/entrypoints/maintenance/test_board_unblock_cov.py +++ b/tests/unit/entrypoints/maintenance/test_board_unblock_cov.py @@ -740,6 +740,20 @@ def test_rule8_policy_blocked_excluded(): assert not _by_rule(actions, "CLEAN_BLOCKED_RETRY") +def test_rule8_backend_capacity_blocked_excluded(): + actions = _run( + [ + _issue( + "1", + state="Blocked", + labels=["task-kind: goal", "blocked-reason: backend-capacity"], + updated_at=_STALE, + ) + ] + ) + assert not _by_rule(actions, "CLEAN_BLOCKED_RETRY") + + # --------------------------------------------------------------------------- # Rule 9 — SPEC_AUTHOR_BACKLOG_PROMOTE # ---------------------------------------------------------------------------