Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions src/operations_center/entrypoints/board_worker/outcomes.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@
# fault, so we never terminate on unknown). See CODE_FAILURE_RETRY_CAP.md.
_CODE_FAILURE_CATEGORIES = frozenset({"validation_failed", "no_changes"})
_TERMINAL_STATE_NAMES = {"cancelled", "done"}
_BACKEND_CAPACITY_REASON_LABEL = "blocked-reason: backend-capacity"
_BACKEND_CAPACITY_REASON_SNIPPETS = (
"stage planner received non-json from agent",
"session limit or error",
)


# ── Atomic failure transition ─────────────────────────────────────────────────
Expand Down Expand Up @@ -97,6 +102,13 @@ def read_improve_output(workspace: Path) -> list[dict]:
return [item for item in raw[:5] if isinstance(item, dict) and item.get("title")]


def _is_backend_capacity_failure(category: str, reason: str) -> bool:
if category != "backend_error":
return False
lowered = reason.lower()
return all(snippet in lowered for snippet in _BACKEND_CAPACITY_REASON_SNIPPETS)


# ── Success handler ───────────────────────────────────────────────────────────


Expand Down Expand Up @@ -510,6 +522,11 @@ def handle_failure(
# 26 policy-blocked re-dispatches in one hour). This label marks the block
# as policy-driven so Rule 8 can exclude it.
add_label(client, issue, "blocked-reason: policy")
elif _is_backend_capacity_failure(category, reason):
# Fallback dispatch hit a backend-capacity/error signature and returned
# no structured planner output; immediate clean-retry recycling just
# burns the next backend slot with the same non-JSON failure.
add_label(client, issue, _BACKEND_CAPACITY_REASON_LABEL)
if executor_signal:
add_label(client, issue, f"executor-signal: {executor_signal}")
if "sigkill" in executor_signal.lower():
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,9 @@
- Not "blocked-reason: policy" (deterministic policy gate — e.g. review.required —
that will re-block identically on every retry; recycling it Blocked->Backlog->
Ready for AI is a closed loop, not a transient infra failure)
- Not "blocked-reason: backend-capacity" (planner/backend returned no structured
output under a backend-capacity condition; immediate recycle just repeats the
same non-JSON/session-limit failure on the next backend slot)
- Blocked for at least --clean-blocked-min-minutes (default 5) minutes
→ move to Backlog for retry.
These represent pre-execution failures (workspace preparation errors, missing
Expand Down Expand Up @@ -268,6 +271,7 @@ def _dispatch_cooldown_reason(
_HANDOFF_IMPROVEMENT_LABEL = "handoff-reason: improvement_applied"
_PR_URL_PREFIX = "pr-url:"
_BLOCKED_REASON_POLICY_LABEL = "blocked-reason: policy"
_BLOCKED_REASON_BACKEND_CAPACITY_LABEL = "blocked-reason: backend-capacity"
_OPEN_PR_GATE_LABEL = "OPEN_PR_GATE"


Expand Down Expand Up @@ -803,6 +807,7 @@ def _apply_rules(
and not _has_label_prefix(labels, "executor-exit-code:")
and not _has_label_prefix(labels, _BLOCKED_BY_PREFIX)
and not _has_label(labels, _BLOCKED_REASON_POLICY_LABEL)
and not _has_label(labels, _BLOCKED_REASON_BACKEND_CAPACITY_LABEL)
)
if is_clean_blocked:
updated_at = _parse_updated_at(issue)
Expand Down
17 changes: 17 additions & 0 deletions tests/unit/entrypoints/board_worker/test_outcomes_cov.py
Original file line number Diff line number Diff line change
Expand Up @@ -711,6 +711,23 @@ def test_handle_failure_non_policy_category_no_policy_label():
assert "blocked-reason: policy" not in _all_update_labels(client)


def test_handle_failure_backend_capacity_adds_structured_label():
client = _make_client()
issue = {"id": "b1", "labels": []}
outcomes.handle_failure(
client,
issue,
"goal",
"goal",
{
"failure_category": "backend_error",
"failure_reason": "Stage planner received non-JSON from agent (session limit or error): ",
},
_make_settings(),
)
assert "blocked-reason: backend-capacity" in _all_update_labels(client)


def test_handle_failure_transient_category_does_not_count():
# Env/transient categories are NOT code failures — must not bump the counter.
for cat in ("backend_error", "timeout", "unknown", "scope_too_wide"):
Expand Down
13 changes: 13 additions & 0 deletions tests/unit/entrypoints/maintenance/test_board_unblock.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,19 @@ def test_rule8_skip_spec_author_too_young():
assert len(retry_actions) == 0


def test_rule8_skip_backend_capacity_blocked():
"""Rule 8 must NOT re-queue backend-capacity failures as clean retries."""
issue = _issue(
"t3b",
state="Blocked",
labels=["task-kind: spec-author", "blocked-reason: backend-capacity"],
updated_at="2026-05-28T10:00:00+00:00",
)
actions = _apply_rules([issue], **_RULES_KWARGS)
retry_actions = [a for a in actions if a["rule"] == "CLEAN_BLOCKED_RETRY"]
assert len(retry_actions) == 0


# --- Rule 9: SPEC_AUTHOR_BACKLOG_PROMOTE ---


Expand Down
14 changes: 14 additions & 0 deletions tests/unit/entrypoints/maintenance/test_board_unblock_cov.py
Original file line number Diff line number Diff line change
Expand Up @@ -740,6 +740,20 @@ def test_rule8_policy_blocked_excluded():
assert not _by_rule(actions, "CLEAN_BLOCKED_RETRY")


def test_rule8_backend_capacity_blocked_excluded():
actions = _run(
[
_issue(
"1",
state="Blocked",
labels=["task-kind: goal", "blocked-reason: backend-capacity"],
updated_at=_STALE,
)
]
)
assert not _by_rule(actions, "CLEAN_BLOCKED_RETRY")


# ---------------------------------------------------------------------------
# Rule 9 — SPEC_AUTHOR_BACKLOG_PROMOTE
# ---------------------------------------------------------------------------
Expand Down
Loading