Skip to content

Commit 0cc9ab9

Browse files
Pigbibicodex
Pigbibi
and
codex
committed
fix: enforce repo-level trusted failure streaks
Co-Authored-By: Codex <noreply@openai.com>
1 parent 59e0ee3 commit 0cc9ab9

3 files changed

Lines changed: 29 additions & 16 deletions

File tree

‎service/automation_decision.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
DEFAULT_LOW_COST_MODEL = "gpt-5.4-mini"
3131
EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH"
3232
POLICY_LOAD_ERROR_KEY = "_load_error"
33+
TRUSTED_FAILURE_ORIGINS = frozenset({"service_job"})
3334
QUOTA_STATUS_SEVERITY = {
3435
"ok": 0,
3536
"healthy": 0,
@@ -154,15 +155,16 @@ def consecutive_failure_count(
154155
repo: str,
155156
task_name: str = "",
156157
) -> int:
157-
"""Count latest consecutive failed runs for one repo/task from newest-first runs."""
158+
"""Count latest consecutive trusted failed runs for one repo from newest-first runs."""
158159
count = 0
159160
normalized_repo = _normalize_repo_id(repo)
160161
for run in runs:
161162
if not isinstance(run, dict):
162163
continue
163-
if normalized_repo and _normalize_repo_id(_repo_from_run(run)) != normalized_repo:
164+
metadata = run.get("metadata") if isinstance(run.get("metadata"), dict) else {}
165+
if str(metadata.get("origin") or "") not in TRUSTED_FAILURE_ORIGINS:
164166
continue
165-
if task_name and str(run.get("task_name") or "") != task_name:
167+
if normalized_repo and _normalize_repo_id(_repo_from_run(run)) != normalized_repo:
166168
continue
167169
state = str(run.get("task_state") or "").strip().lower()
168170
if state == "failed":

‎tests/test_ai_gateway_automation_control.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ def test_control_snapshot_scans_full_retained_ledger_for_repo_failure_streak(sel
8787
{
8888
"task_name": f"other-{index}",
8989
"task_state": "merged",
90-
"metadata": {"source_repository": f"QuantStrategyLab/Other{index}"},
90+
"metadata": {"origin": "service_job", "source_repository": f"QuantStrategyLab/Other{index}"},
9191
}
9292
for index in range(20)
9393
]
@@ -96,12 +96,12 @@ def test_control_snapshot_scans_full_retained_ledger_for_repo_failure_streak(sel
9696
{
9797
"task_name": "monthly",
9898
"task_state": "failed",
99-
"metadata": {"source_repository": "QuantStrategyLab/TargetRepo"},
99+
"metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"},
100100
},
101101
{
102102
"task_name": "monthly",
103103
"task_state": "failed",
104-
"metadata": {"source_repository": "QuantStrategyLab/TargetRepo"},
104+
"metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"},
105105
},
106106
]
107107
)
@@ -137,7 +137,7 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None
137137
"run_id": "previous-run",
138138
"task_name": "monthly",
139139
"task_state": "failed",
140-
"metadata": {"source_repository": "QuantStrategyLab/TargetRepo"},
140+
"metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"},
141141
}
142142
]
143143
health = type("Health", (), {"status": "healthy"})()
@@ -147,7 +147,7 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None
147147
"run_id": "current-run",
148148
"task_name": "monthly",
149149
"task_state": "failed",
150-
"metadata": {"source_repository": "QuantStrategyLab/TargetRepo"},
150+
"metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"},
151151
}
152152

153153
with (
@@ -174,7 +174,7 @@ def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None:
174174
"run_id": "current-run",
175175
"task_name": "monthly",
176176
"task_state": "failed",
177-
"metadata": {"source_repository": "QuantStrategyLab/TargetRepo"},
177+
"metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"},
178178
}
179179
]
180180
health = type("Health", (), {"status": "healthy"})()
@@ -184,7 +184,7 @@ def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None:
184184
"run_id": "current-run",
185185
"task_name": "monthly",
186186
"task_state": "failed",
187-
"metadata": {"source_repository": "QuantStrategyLab/TargetRepo"},
187+
"metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"},
188188
}
189189

190190
with (

‎tests/test_automation_decision.py‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -128,12 +128,23 @@ def test_failure_streak_matching_is_case_insensitive(self) -> None:
128128
{
129129
"task_name": "monthly",
130130
"task_state": "failed",
131-
"metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"},
131+
"metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"},
132132
},
133133
]
134134

135135
self.assertEqual(consecutive_failure_count(runs, repo="quantstrategylab/aiauditbridge", task_name="monthly"), 1)
136136

137+
def test_external_workflow_failures_do_not_force_repo_failure_streak(self) -> None:
138+
runs = [
139+
{
140+
"task_name": "monthly",
141+
"task_state": "failed",
142+
"metadata": {"origin": "external_workflow", "source_repository": "QuantStrategyLab/AIAuditBridge"},
143+
},
144+
]
145+
146+
self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 0)
147+
137148
def test_invalid_repo_autonomy_fails_closed(self) -> None:
138149
result = decide_automation_execution(
139150
repo="QuantStrategyLab/CryptoLivePoolPipelines",
@@ -154,12 +165,12 @@ def test_consecutive_failures_force_human_review(self) -> None:
154165
{
155166
"task_name": "monthly",
156167
"task_state": "failed",
157-
"metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"},
168+
"metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"},
158169
},
159170
{
160-
"task_name": "monthly",
171+
"task_name": "runtime-health",
161172
"task_state": "failed",
162-
"metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"},
173+
"metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"},
163174
},
164175
]
165176

@@ -184,12 +195,12 @@ def test_running_state_does_not_clear_failure_streak(self) -> None:
184195
{
185196
"task_name": "monthly",
186197
"task_state": "running",
187-
"metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"},
198+
"metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"},
188199
},
189200
{
190201
"task_name": "monthly",
191202
"task_state": "failed",
192-
"metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"},
203+
"metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"},
193204
},
194205
]
195206

0 commit comments

Comments
 (0)