From 6936e970fa72b46fb6ab1d02a8a515ae2a415e46 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 02:49:39 +0800 Subject: [PATCH 01/72] feat: add health-driven automation decisions Co-Authored-By: Codex --- docs/ai_autonomy_architecture.md | 9 + docs/async_service_deployment.md | 20 ++ scripts/deploy_codex_audit_service.sh | 4 +- service/ai_gateway_service.py | 27 ++- service/automation_decision.py | 218 ++++++++++++++++++++ tests/test_ai_gateway_automation_control.py | 51 +++++ tests/test_ai_gateway_service_get_routes.py | 4 + tests/test_automation_decision.py | 138 +++++++++++++ tests/test_run_monthly_codex_audit.py | 1 + 9 files changed, 466 insertions(+), 6 deletions(-) create mode 100644 service/automation_decision.py create mode 100644 tests/test_ai_gateway_automation_control.py create mode 100644 tests/test_automation_decision.py diff --git a/docs/ai_autonomy_architecture.md b/docs/ai_autonomy_architecture.md index 41902644..0d2bd615 100644 --- a/docs/ai_autonomy_architecture.md +++ b/docs/ai_autonomy_architecture.md @@ -337,6 +337,15 @@ AIAuditBridge 是 QuantStrategyLab 的 AI 审计控制面，负责： - 健康驱动的执行降级策略； - repo 级别自治阈值。 +首批落地边界： + +- `/v1/ai/automation/control` 输出 `execution` 决策快照； +- health degraded / runtime pause 时，执行模式降级到 `review_only`； +- quota low 时给出低成本模型建议，quota exhausted / blocked 时建议 defer； +- 连续失败达到 repo 阈值时强制 human review； +- repo 级 `max_autonomy` / `max_consecutive_failures` 从服务端受控 policy 文件读取，不信任被审仓库 checkout； +- 该阶段只影响调度建议和控制面输出，不自动放宽 merge / deploy 权限。 + ### Phase 4：扩大自动修复，但只扩大低风险面 目标：提升无人值守覆盖率，但不放松安全门。 diff --git a/docs/async_service_deployment.md b/docs/async_service_deployment.md index 4a1a6b53..cd36014b 100644 --- a/docs/async_service_deployment.md +++ b/docs/async_service_deployment.md @@ -71,6 +71,26 @@ bash scripts/deploy_codex_audit_service.sh deploy ``` The job directory should be owned by the service user and mode `0700`. +The deploy script points `CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH` to +`${CODEX_AUDIT_SERVICE_JOB_DIR}/execution_policy.json`. If present, this +service-owned file can cap repo autonomy without trusting the reviewed checkout: + +```json +{ + "default": { + "max_autonomy": "auto_pr", + "max_consecutive_failures": 3, + "low_cost_model": "gpt-5.4-mini" + }, + "repositories": { + "QuantStrategyLab/CryptoLivePoolPipelines": { + "max_autonomy": "review_only", + "max_consecutive_failures": 2 + } + } +} +``` + The service should rely on an authenticated Codex CLI session and must not inject OpenAI/Codex API keys into the Codex subprocess. With `CODEX_AUDIT_SERVICE_CODEX_ACCOUNT_USAGE=1`, `/v1/ai/quota` includes a diff --git a/scripts/deploy_codex_audit_service.sh b/scripts/deploy_codex_audit_service.sh index 0e9fb05c..52705557 100644 --- a/scripts/deploy_codex_audit_service.sh +++ b/scripts/deploy_codex_audit_service.sh @@ -60,7 +60,7 @@ systemctl_environment_brief() { | sed 's/^Environment=//' \ | tr ' ' '\n' \ | sed -E "s/^[\"']//; s/[\"']$//" \ - | grep -E '^CODEX_AUDIT_SERVICE_(ALLOWED_|AUDIENCE=|HOST=|PORT=|JOB_DIR=|QUOTA_STORE=|CODEX_ACCOUNT_USAGE=|OPENAI_USAGE_WINDOW_DAYS=|ANTHROPIC_USAGE_WINDOW_DAYS=|SANDBOX=|MODEL=|REASONING_EFFORT=)' \ + | grep -E '^CODEX_AUDIT_SERVICE_(ALLOWED_|AUDIENCE=|HOST=|PORT=|JOB_DIR=|QUOTA_STORE=|EXECUTION_POLICY_PATH=|CODEX_ACCOUNT_USAGE=|OPENAI_USAGE_WINDOW_DAYS=|ANTHROPIC_USAGE_WINDOW_DAYS=|SANDBOX=|MODEL=|REASONING_EFFORT=)' \ | mask_infra || true fi } @@ -252,6 +252,7 @@ Environment=CODEX_AUDIT_SERVICE_ALLOWED_REPOSITORY_VISIBILITIES=${ALLOWED_REPOSI Environment=CODEX_AUDIT_SERVICE_ALLOWED_SOURCE_REPOSITORIES=${ALLOWED_SOURCE_REPOSITORIES} Environment=CODEX_AUDIT_SERVICE_JOB_DIR=${JOB_DIR} Environment=CODEX_AUDIT_SERVICE_QUOTA_STORE=${JOB_DIR}/quota.json +Environment=CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH=${JOB_DIR}/execution_policy.json Environment=CODEX_AUDIT_SERVICE_CODEX_ACCOUNT_USAGE=${CODEX_ACCOUNT_USAGE} Environment=CODEX_AUDIT_SERVICE_OPENAI_USAGE_WINDOW_DAYS=${OPENAI_USAGE_WINDOW_DAYS} Environment=CODEX_AUDIT_SERVICE_ANTHROPIC_USAGE_WINDOW_DAYS=${ANTHROPIC_USAGE_WINDOW_DAYS} @@ -281,6 +282,7 @@ Environment="CODEX_AUDIT_SERVICE_ALLOWED_WORKFLOW_REFS=${ALLOWED_WORKFLOW_REFS}" Environment="CODEX_AUDIT_SERVICE_ALLOWED_REFS=${ALLOWED_REFS}" Environment="CODEX_AUDIT_SERVICE_ALLOWED_REPOSITORY_VISIBILITIES=${ALLOWED_REPOSITORY_VISIBILITIES}" Environment="CODEX_AUDIT_SERVICE_ALLOWED_SOURCE_REPOSITORIES=${ALLOWED_SOURCE_REPOSITORIES}" +Environment="CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH=${JOB_DIR}/execution_policy.json" EOF_DROPIN } diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index c506727e..f933c90e 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -72,6 +72,7 @@ get_automation_run_ledger, suggest_control_action, ) +from service.automation_decision import decide_automation_execution, load_execution_policy from service.strategy_automation_registry import ( apply_strategy_registry_guard, summarize_strategy_registry_context, @@ -507,7 +508,7 @@ def _public_job_payload(job: dict[str, Any]) -> dict[str, object]: return payload -def _automation_control_snapshot(repo: str) -> dict[str, Any]: +def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mode: str = "review_and_fix") -> dict[str, Any]: try: org_health = read_org_health() except Exception: @@ -516,7 +517,23 @@ def _automation_control_snapshot(repo: str) -> dict[str, Any]: quota_status = get_quota_manager().runtime_status(repo or "unknown") except Exception: quota_status = {"status": "unavailable"} - return suggest_control_action(get_health_monitor().status, quota_status, org_health) + control = suggest_control_action(get_health_monitor().status, quota_status, org_health) + try: + recent_runs = get_automation_run_ledger().snapshot(limit=20)["runs"] + except Exception: + recent_runs = [] + control["execution"] = decide_automation_execution( + repo=repo or "unknown", + task_name=task_name, + requested_mode=requested_mode, + control_action=str(control.get("action") or CONTROL_REVIEW_ONLY), + service_health=control.get("service_health"), + quota_status=quota_status, + org_health_status=control.get("org_health_status"), + recent_runs=recent_runs, + policy=load_execution_policy(), + ) + return control def _highest_changed_path_risk(changed_paths: list[str], policy: dict[str, Any]) -> str: @@ -540,7 +557,7 @@ def _automation_triage_snapshot( changed_paths: list[str] | None = None, run_id: str = "", ) -> dict[str, Any]: - control = _automation_control_snapshot(repo) + control = _automation_control_snapshot(repo, task_name=task) policy = load_autonomy_policy() normalized_paths = [ normalized @@ -648,7 +665,7 @@ def _automation_triage_snapshot( def _record_job_automation_run(job: dict[str, Any]) -> None: try: repo = str(job.get("source_repository") or job.get("repository") or "unknown") - control = _automation_control_snapshot(repo) + control = _automation_control_snapshot(repo, task_name=str(job.get("task") or ""), requested_mode=str(job.get("mode") or "review_and_fix")) get_automation_run_ledger().record( str(job.get("job_id") or ""), job_task_state(job), @@ -1504,7 +1521,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st _validate_source_repo_org(claims, source_repo) _assert_source_repository_owner_or_operator(claims, source_repo) repo = source_repo or str(claims.get("repository") or "unknown") - control = _automation_control_snapshot(repo) + control = _automation_control_snapshot(repo, task_name=str(payload.get("task") or payload.get("task_name") or "")) metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {} ledger = get_automation_run_ledger() run_id = str(payload.get("run_id") or payload.get("job_id") or "") diff --git a/service/automation_decision.py b/service/automation_decision.py new file mode 100644 index 00000000..393967cb --- /dev/null +++ b/service/automation_decision.py @@ -0,0 +1,218 @@ +"""Health-driven execution decisions for automation scheduling.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +from service.automation_run_ledger import CONTROL_ESCALATE, CONTROL_PAUSE_AUTO_FIX, CONTROL_REVIEW_ONLY +from service.quota import recommend_model + +EXECUTION_RUN = "run" +EXECUTION_REVIEW_ONLY = "review_only" +EXECUTION_DEFER = "defer" +EXECUTION_HUMAN_REVIEW = "human_review" + +MODE_REVIEW_AND_FIX = "review_and_fix" +MODE_REVIEW_ONLY = "review_only" + +AUTONOMY_MANUAL = "manual" +AUTONOMY_REVIEW_ONLY = "review_only" +AUTONOMY_AUTO_PR = "auto_pr" +AUTONOMY_AUTO_MERGE = "auto_merge" +AUTONOMY_ORDER = (AUTONOMY_MANUAL, AUTONOMY_REVIEW_ONLY, AUTONOMY_AUTO_PR, AUTONOMY_AUTO_MERGE) +AUTONOMY_RANK = {level: index for index, level in enumerate(AUTONOMY_ORDER)} + +DEFAULT_MAX_CONSECUTIVE_FAILURES = 3 +DEFAULT_LOW_COST_MODEL = "gpt-5.4-mini" +EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH" +QUOTA_STATUS_SEVERITY = { + "ok": 0, + "healthy": 0, + "unknown": 1, + "unavailable": 1, + "low": 2, + "constrained": 2, + "exhausted": 3, + "blocked": 3, +} + + +def _normalize_status(value: Any, default: str = "unknown") -> str: + if isinstance(value, dict): + value = value.get("status", default) + return str(value or default).strip().lower() + + +def _normalize_quota_status(value: Any, default: str = "unknown") -> str: + statuses = [_normalize_status(value, "")] + if isinstance(value, dict) and isinstance(value.get("quota"), dict): + statuses.append(_normalize_status(value["quota"], "")) + normalized = [status for status in statuses if status] + if not normalized: + return default + return max(normalized, key=lambda status: QUOTA_STATUS_SEVERITY.get(status, 1)) + + +def _normalize_mode(value: str) -> str: + return MODE_REVIEW_AND_FIX if str(value or "").strip() == MODE_REVIEW_AND_FIX else MODE_REVIEW_ONLY + + +def _normalize_autonomy(value: Any, default: str = AUTONOMY_AUTO_PR) -> str: + level = str(value or default).strip().lower() + return level if level in AUTONOMY_RANK else default + + +def _repo_from_run(run: dict[str, Any]) -> str: + metadata = run.get("metadata") if isinstance(run.get("metadata"), dict) else {} + return str(metadata.get("source_repository") or metadata.get("repository") or "") + + +def load_execution_policy(path: Path | None = None) -> dict[str, Any]: + """Load service-owned execution policy for repo autonomy thresholds.""" + if path is None: + configured = os.environ.get(EXECUTION_POLICY_PATH_ENV, "").strip() + if not configured: + return {} + path = Path(configured).expanduser() + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return payload if isinstance(payload, dict) else {} + + +def repo_execution_policy(repo: str, policy: dict[str, Any] | None = None) -> dict[str, Any]: + """Merge default and repo-specific execution policy without trusting repo checkouts.""" + raw = policy if isinstance(policy, dict) else {} + defaults = raw.get("default") if isinstance(raw.get("default"), dict) else {} + repositories = raw.get("repositories") if isinstance(raw.get("repositories"), dict) else {} + override = repositories.get(repo) if isinstance(repositories.get(repo), dict) else {} + return {**defaults, **override} + + +def consecutive_failure_count( + runs: list[dict[str, Any]], + *, + repo: str, + task_name: str = "", +) -> int: + """Count latest consecutive failed runs for one repo/task from newest-first runs.""" + count = 0 + for run in runs: + if not isinstance(run, dict): + continue + if repo and _repo_from_run(run) != repo: + continue + if task_name and str(run.get("task_name") or "") != task_name: + continue + state = str(run.get("task_state") or "").strip().lower() + if state == "failed": + count += 1 + continue + if state: + break + return count + + +def decide_automation_execution( + *, + repo: str, + task_name: str = "", + requested_mode: str = MODE_REVIEW_AND_FIX, + requested_provider: str = "auto", + requested_model: str = "", + control_action: str = CONTROL_REVIEW_ONLY, + service_health: Any = "", + quota_status: Any = "", + org_health_status: Any = "", + recent_runs: list[dict[str, Any]] | None = None, + policy: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Produce a safe execution decision from health, quota, failures, and repo policy.""" + repo_policy = repo_execution_policy(repo, policy) + max_autonomy = _normalize_autonomy(repo_policy.get("max_autonomy"), AUTONOMY_AUTO_PR) + max_failures = int(repo_policy.get("max_consecutive_failures") or DEFAULT_MAX_CONSECUTIVE_FAILURES) + low_cost_model = str(repo_policy.get("low_cost_model") or DEFAULT_LOW_COST_MODEL) + quota_low_behavior = str(repo_policy.get("quota_low_behavior") or "low_cost_model").strip().lower() + + effective_mode = _normalize_mode(requested_mode) + effective_provider = str(requested_provider or "auto").strip().lower() or "auto" + effective_model = str(requested_model or "").strip() + action = EXECUTION_RUN + reasons: list[str] = [] + human_review_required = False + defer = False + + service = _normalize_status(service_health) + quota = _normalize_quota_status(quota_status) + org_health = _normalize_status(org_health_status) + failures = consecutive_failure_count(recent_runs or [], repo=repo, task_name=task_name) + + if AUTONOMY_RANK[max_autonomy] <= AUTONOMY_RANK[AUTONOMY_REVIEW_ONLY]: + effective_mode = MODE_REVIEW_ONLY + human_review_required = True + reasons.append(f"repo max autonomy is {max_autonomy}") + if max_autonomy == AUTONOMY_MANUAL: + action = EXECUTION_HUMAN_REVIEW + + if failures >= max_failures: + action = EXECUTION_HUMAN_REVIEW + effective_mode = MODE_REVIEW_ONLY + human_review_required = True + reasons.append(f"consecutive failures reached {failures}/{max_failures}") + + if control_action in {CONTROL_REVIEW_ONLY, CONTROL_PAUSE_AUTO_FIX, CONTROL_ESCALATE}: + effective_mode = MODE_REVIEW_ONLY + human_review_required = True + reasons.append(f"runtime control action is {control_action}") + if control_action == CONTROL_ESCALATE: + action = EXECUTION_HUMAN_REVIEW + + if service == "degraded" or org_health == "degraded": + effective_mode = MODE_REVIEW_ONLY + human_review_required = True + reasons.append("health degraded; forcing review_only") + if service == "unhealthy" or org_health == "unhealthy": + action = EXECUTION_HUMAN_REVIEW + effective_mode = MODE_REVIEW_ONLY + human_review_required = True + reasons.append("health unhealthy; forcing human review") + + if quota in {"low", "constrained"}: + effective_model = effective_model or low_cost_model or recommend_model(0.0) + if quota_low_behavior == "defer": + action = EXECUTION_DEFER + defer = True + effective_mode = MODE_REVIEW_ONLY + human_review_required = True + reasons.append(f"quota status is {quota}; deferring automation") + else: + reasons.append(f"quota status is {quota}; recommending low-cost model") + elif quota in {"exhausted", "blocked"}: + action = EXECUTION_DEFER + defer = True + effective_mode = MODE_REVIEW_ONLY + human_review_required = True + reasons.append(f"quota status is {quota}; deferring automation") + + return { + "action": action, + "repo": repo, + "task_name": task_name, + "requested_mode": _normalize_mode(requested_mode), + "effective_mode": effective_mode, + "requested_provider": requested_provider, + "effective_provider": effective_provider, + "requested_model": requested_model, + "effective_model": effective_model, + "max_autonomy": max_autonomy, + "consecutive_failures": failures, + "max_consecutive_failures": max_failures, + "human_review_required": human_review_required, + "auto_fix_allowed": action == EXECUTION_RUN and effective_mode == MODE_REVIEW_AND_FIX and not human_review_required, + "defer": defer, + "reasons": reasons or ["execution allowed"], + } diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py new file mode 100644 index 00000000..24153724 --- /dev/null +++ b/tests/test_ai_gateway_automation_control.py @@ -0,0 +1,51 @@ +"""Tests for automation control-plane execution decisions.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest +from unittest.mock import patch + +from service.ai_gateway_service import _automation_control_snapshot + + +class TestAutomationControlSnapshot(unittest.TestCase): + def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: + with TemporaryDirectory() as tmp: + policy_path = Path(tmp) / "execution_policy.json" + policy_path.write_text( + json.dumps( + { + "repositories": { + "QuantStrategyLab/TargetRepo": { + "max_autonomy": "review_only", + "max_consecutive_failures": 2, + } + } + } + ), + encoding="utf-8", + ) + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type("Ledger", (), {"snapshot": lambda self, limit=20: {"runs": []}})() + + with ( + patch.dict(os.environ, {"CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH": str(policy_path)}, clear=False), + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") + + self.assertEqual(control["action"], "continue") + self.assertEqual(control["execution"]["effective_mode"], "review_only") + self.assertFalse(control["execution"]["auto_fix_allowed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index d83c062a..c46e450b 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -372,6 +372,9 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None ) with urllib.request.urlopen(request, timeout=5) as response: self.assertEqual(response.status, 200) + control = json.loads(response.read().decode("utf-8"))["control"] + self.assertIn("execution", control) + self.assertEqual(control["execution"]["repo"], "QuantStrategyLab/TargetRepo") missing_repo_request = urllib.request.Request( f"{base_url}/v1/ai/automation/control", @@ -576,6 +579,7 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: with urllib.request.urlopen(f"{base_url}/v1/ai/automation/control?repo=local/repo", timeout=5) as response: control = json.loads(response.read().decode("utf-8"))["control"] self.assertIn(control["action"], {"continue", "review_only", "pause_auto_fix", "escalate"}) + self.assertIn("execution", control) finally: server.shutdown() server.server_close() diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py new file mode 100644 index 00000000..33a091d2 --- /dev/null +++ b/tests/test_automation_decision.py @@ -0,0 +1,138 @@ +"""Tests for health-driven automation execution decisions.""" + +from __future__ import annotations + +import json +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +from service.automation_decision import ( + EXECUTION_DEFER, + EXECUTION_HUMAN_REVIEW, + EXECUTION_RUN, + MODE_REVIEW_AND_FIX, + MODE_REVIEW_ONLY, + consecutive_failure_count, + decide_automation_execution, + load_execution_policy, +) +from service.automation_run_ledger import CONTROL_CONTINUE, CONTROL_ESCALATE, CONTROL_PAUSE_AUTO_FIX + + +class TestAutomationDecision(unittest.TestCase): + def test_healthy_control_allows_review_and_fix(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode=MODE_REVIEW_AND_FIX, + control_action=CONTROL_CONTINUE, + service_health="healthy", + quota_status="ok", + org_health_status="ok", + ) + + self.assertEqual(result["action"], EXECUTION_RUN) + self.assertEqual(result["effective_mode"], MODE_REVIEW_AND_FIX) + self.assertTrue(result["auto_fix_allowed"]) + + def test_degraded_health_forces_review_only(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode=MODE_REVIEW_AND_FIX, + control_action=CONTROL_PAUSE_AUTO_FIX, + service_health="degraded", + quota_status="ok", + org_health_status="ok", + ) + + self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) + self.assertFalse(result["auto_fix_allowed"]) + self.assertTrue(result["human_review_required"]) + + def test_exhausted_quota_defers_execution(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode=MODE_REVIEW_AND_FIX, + control_action=CONTROL_ESCALATE, + service_health="healthy", + quota_status={"status": "exhausted"}, + org_health_status="ok", + ) + + self.assertEqual(result["action"], EXECUTION_DEFER) + self.assertTrue(result["defer"]) + self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) + + def test_low_quota_recommends_low_cost_model(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode=MODE_REVIEW_AND_FIX, + control_action=CONTROL_CONTINUE, + service_health="healthy", + quota_status="low", + org_health_status="ok", + policy={"default": {"low_cost_model": "gpt-5.4-mini"}}, + ) + + self.assertEqual(result["action"], EXECUTION_RUN) + self.assertEqual(result["effective_model"], "gpt-5.4-mini") + self.assertTrue(any("low-cost model" in reason for reason in result["reasons"])) + + def test_repo_policy_can_force_review_only(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/CryptoLivePoolPipelines", + requested_mode=MODE_REVIEW_AND_FIX, + control_action=CONTROL_CONTINUE, + service_health="healthy", + quota_status="ok", + org_health_status="ok", + policy={"repositories": {"QuantStrategyLab/CryptoLivePoolPipelines": {"max_autonomy": "review_only"}}}, + ) + + self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) + self.assertTrue(result["human_review_required"]) + self.assertFalse(result["auto_fix_allowed"]) + + def test_consecutive_failures_force_human_review(self) -> None: + runs = [ + { + "task_name": "monthly", + "task_state": "failed", + "metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"}, + }, + { + "task_name": "monthly", + "task_state": "failed", + "metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"}, + }, + ] + + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + task_name="monthly", + requested_mode=MODE_REVIEW_AND_FIX, + control_action=CONTROL_CONTINUE, + service_health="healthy", + quota_status="ok", + org_health_status="ok", + recent_runs=runs, + policy={"default": {"max_consecutive_failures": 2}}, + ) + + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge", task_name="monthly"), 2) + self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) + self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) + + def test_load_execution_policy_ignores_malformed_files(self) -> None: + with TemporaryDirectory() as tmp: + path = Path(tmp) / "policy.json" + path.write_text("not-json", encoding="utf-8") + + self.assertEqual(load_execution_policy(path), {}) + + path.write_text(json.dumps({"default": {"max_autonomy": "review_only"}}), encoding="utf-8") + self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "review_only") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_run_monthly_codex_audit.py b/tests/test_run_monthly_codex_audit.py index da89e57e..644d6748 100644 --- a/tests/test_run_monthly_codex_audit.py +++ b/tests/test_run_monthly_codex_audit.py @@ -2342,6 +2342,7 @@ def test_vps_deploy_adds_nginx_audit_route_without_router_service(self) -> None: self.assertIn("location = /v1/codex-audit", deploy_script) self.assertIn("location ^~ /v1/codex-audit/", deploy_script) self.assertIn("CODEX_AUDIT_SERVICE_JOB_DIR", deploy_script) + self.assertIn("CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH", deploy_script) self.assertIn("codex_pr_review.yml@refs/pull/*/merge", deploy_script) self.assertIn("refs/pull/*/merge", deploy_script) self.assertIn("QuantStrategyLab/AIAuditBridge,QuantStrategyLab/CryptoLivePoolPipelines", deploy_script) From 8e66743757a633df976d8bcc19baa9814de0e0f5 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 02:57:34 +0800 Subject: [PATCH 02/72] fix: harden automation execution control Co-Authored-By: Codex --- service/ai_gateway_service.py | 15 ++++--- service/automation_decision.py | 10 ++++- tests/test_ai_gateway_automation_control.py | 48 +++++++++++++++++++++ tests/test_ai_gateway_service_get_routes.py | 2 + tests/test_automation_decision.py | 14 ++++++ 5 files changed, 83 insertions(+), 6 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index f933c90e..3d9b0ba0 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -508,7 +508,7 @@ def _public_job_payload(job: dict[str, Any]) -> dict[str, object]: return payload -def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mode: str = "review_and_fix") -> dict[str, Any]: +def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mode: str = MODE_REVIEW_ONLY) -> dict[str, Any]: try: org_health = read_org_health() except Exception: @@ -519,7 +519,7 @@ def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mo quota_status = {"status": "unavailable"} control = suggest_control_action(get_health_monitor().status, quota_status, org_health) try: - recent_runs = get_automation_run_ledger().snapshot(limit=20)["runs"] + recent_runs = get_automation_run_ledger().snapshot(limit=None)["runs"] except Exception: recent_runs = [] control["execution"] = decide_automation_execution( @@ -665,7 +665,7 @@ def _automation_triage_snapshot( def _record_job_automation_run(job: dict[str, Any]) -> None: try: repo = str(job.get("source_repository") or job.get("repository") or "unknown") - control = _automation_control_snapshot(repo, task_name=str(job.get("task") or ""), requested_mode=str(job.get("mode") or "review_and_fix")) + control = _automation_control_snapshot(repo, task_name=str(job.get("task") or ""), requested_mode=str(job.get("mode") or MODE_REVIEW_ONLY)) get_automation_run_ledger().record( str(job.get("job_id") or ""), job_task_state(job), @@ -1443,7 +1443,8 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - _json_response(self, HTTPStatus.OK, {"status": "ok", "control": _automation_control_snapshot(repo)}) + mode = str(params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY) + _json_response(self, HTTPStatus.OK, {"status": "ok", "control": _automation_control_snapshot(repo, requested_mode=mode)}) def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, Any]) -> None: from urllib.parse import parse_qs, urlparse @@ -1521,7 +1522,11 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st _validate_source_repo_org(claims, source_repo) _assert_source_repository_owner_or_operator(claims, source_repo) repo = source_repo or str(claims.get("repository") or "unknown") - control = _automation_control_snapshot(repo, task_name=str(payload.get("task") or payload.get("task_name") or "")) + control = _automation_control_snapshot( + repo, + task_name=str(payload.get("task") or payload.get("task_name") or ""), + requested_mode=str(payload.get("mode") or MODE_REVIEW_ONLY), + ) metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {} ledger = get_automation_run_ledger() run_id = str(payload.get("run_id") or payload.get("job_id") or "") diff --git a/service/automation_decision.py b/service/automation_decision.py index 393967cb..17475be5 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -65,6 +65,14 @@ def _normalize_autonomy(value: Any, default: str = AUTONOMY_AUTO_PR) -> str: return level if level in AUTONOMY_RANK else default +def _safe_positive_int(value: Any, default: int) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + return default + return parsed if parsed > 0 else default + + def _repo_from_run(run: dict[str, Any]) -> str: metadata = run.get("metadata") if isinstance(run.get("metadata"), dict) else {} return str(metadata.get("source_repository") or metadata.get("repository") or "") @@ -134,7 +142,7 @@ def decide_automation_execution( """Produce a safe execution decision from health, quota, failures, and repo policy.""" repo_policy = repo_execution_policy(repo, policy) max_autonomy = _normalize_autonomy(repo_policy.get("max_autonomy"), AUTONOMY_AUTO_PR) - max_failures = int(repo_policy.get("max_consecutive_failures") or DEFAULT_MAX_CONSECUTIVE_FAILURES) + max_failures = _safe_positive_int(repo_policy.get("max_consecutive_failures"), DEFAULT_MAX_CONSECUTIVE_FAILURES) low_cost_model = str(repo_policy.get("low_cost_model") or DEFAULT_LOW_COST_MODEL) quota_low_behavior = str(repo_policy.get("quota_low_behavior") or "low_cost_model").strip().lower() diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 24153724..7ff0ad12 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -46,6 +46,54 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) + def test_control_snapshot_scans_full_retained_ledger_for_repo_failure_streak(self) -> None: + runs = [ + { + "task_name": f"other-{index}", + "task_state": "merged", + "metadata": {"source_repository": f"QuantStrategyLab/Other{index}"}, + } + for index in range(20) + ] + runs.extend( + [ + { + "task_name": "monthly", + "task_state": "failed", + "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + }, + { + "task_name": "monthly", + "task_state": "failed", + "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + }, + ] + ) + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + + class Ledger: + requested_limit = object() + + def snapshot(self, limit=100): + self.requested_limit = limit + return {"runs": runs} + + ledger = Ledger() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={"default": {"max_consecutive_failures": 2}}), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") + + self.assertIsNone(ledger.requested_limit) + self.assertEqual(control["execution"]["action"], "human_review") + self.assertEqual(control["execution"]["consecutive_failures"], 2) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index c46e450b..c1b6ea36 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -375,6 +375,8 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None control = json.loads(response.read().decode("utf-8"))["control"] self.assertIn("execution", control) self.assertEqual(control["execution"]["repo"], "QuantStrategyLab/TargetRepo") + self.assertEqual(control["execution"]["effective_mode"], "review_only") + self.assertFalse(control["execution"]["auto_fix_allowed"]) missing_repo_request = urllib.request.Request( f"{base_url}/v1/ai/automation/control", diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 33a091d2..35924385 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -123,6 +123,20 @@ def test_consecutive_failures_force_human_review(self) -> None: self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) + def test_invalid_failure_threshold_falls_back_safely(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode=MODE_REVIEW_AND_FIX, + control_action=CONTROL_CONTINUE, + service_health="healthy", + quota_status="ok", + org_health_status="ok", + policy={"default": {"max_consecutive_failures": "oops"}}, + ) + + self.assertEqual(result["max_consecutive_failures"], 3) + self.assertEqual(result["action"], EXECUTION_RUN) + def test_load_execution_policy_ignores_malformed_files(self) -> None: with TemporaryDirectory() as tmp: path = Path(tmp) / "policy.json" From f5fe332b59fd29d31455064b9edd85bae3ba9fa1 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:03:44 +0800 Subject: [PATCH 03/72] fix: fail closed automation control policy Co-Authored-By: Codex --- service/ai_gateway_service.py | 20 +++++++++++-- service/automation_decision.py | 17 +++++++---- tests/test_ai_gateway_automation_control.py | 3 +- tests/test_automation_decision.py | 31 +++++++++++++++++++++ 4 files changed, 63 insertions(+), 8 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 3d9b0ba0..2618daa0 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -72,7 +72,7 @@ get_automation_run_ledger, suggest_control_action, ) -from service.automation_decision import decide_automation_execution, load_execution_policy +from service.automation_decision import EXECUTION_DEFER, EXECUTION_HUMAN_REVIEW, decide_automation_execution, load_execution_policy from service.strategy_automation_registry import ( apply_strategy_registry_guard, summarize_strategy_registry_context, @@ -522,7 +522,7 @@ def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mo recent_runs = get_automation_run_ledger().snapshot(limit=None)["runs"] except Exception: recent_runs = [] - control["execution"] = decide_automation_execution( + execution = decide_automation_execution( repo=repo or "unknown", task_name=task_name, requested_mode=requested_mode, @@ -533,6 +533,22 @@ def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mo recent_runs=recent_runs, policy=load_execution_policy(), ) + original_action = str(control.get("action") or CONTROL_REVIEW_ONLY) + strict_action = original_action + if execution.get("action") == EXECUTION_HUMAN_REVIEW: + strict_action = CONTROL_ESCALATE + elif execution.get("action") == EXECUTION_DEFER: + strict_action = CONTROL_PAUSE_AUTO_FIX + elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: + strict_action = CONTROL_REVIEW_ONLY + if strict_action != original_action: + control["action"] = strict_action + control["requires_human_review"] = True + control["auto_fix_allowed"] = False + reasons = control.get("reasons") if isinstance(control.get("reasons"), list) else [] + reasons.append("capped by execution decision") + control["reasons"] = reasons + control["execution"] = execution return control diff --git a/service/automation_decision.py b/service/automation_decision.py index 17475be5..4c2d24fe 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -9,6 +9,7 @@ from service.automation_run_ledger import CONTROL_ESCALATE, CONTROL_PAUSE_AUTO_FIX, CONTROL_REVIEW_ONLY from service.quota import recommend_model +from service.task_state import TERMINAL_STATES EXECUTION_RUN = "run" EXECUTION_REVIEW_ONLY = "review_only" @@ -60,9 +61,13 @@ def _normalize_mode(value: str) -> str: return MODE_REVIEW_AND_FIX if str(value or "").strip() == MODE_REVIEW_AND_FIX else MODE_REVIEW_ONLY -def _normalize_autonomy(value: Any, default: str = AUTONOMY_AUTO_PR) -> str: - level = str(value or default).strip().lower() - return level if level in AUTONOMY_RANK else default +def _parse_autonomy(value: Any, default: str = AUTONOMY_AUTO_PR) -> tuple[str, str]: + level = str(value or "").strip().lower() + if not level: + return default, "" + if level in AUTONOMY_RANK: + return level, "" + return AUTONOMY_MANUAL, f"invalid max_autonomy {level!r}; forcing manual" def _safe_positive_int(value: Any, default: int) -> int: @@ -120,7 +125,7 @@ def consecutive_failure_count( if state == "failed": count += 1 continue - if state: + if state in TERMINAL_STATES: break return count @@ -141,7 +146,7 @@ def decide_automation_execution( ) -> dict[str, Any]: """Produce a safe execution decision from health, quota, failures, and repo policy.""" repo_policy = repo_execution_policy(repo, policy) - max_autonomy = _normalize_autonomy(repo_policy.get("max_autonomy"), AUTONOMY_AUTO_PR) + max_autonomy, autonomy_config_error = _parse_autonomy(repo_policy.get("max_autonomy"), AUTONOMY_AUTO_PR) max_failures = _safe_positive_int(repo_policy.get("max_consecutive_failures"), DEFAULT_MAX_CONSECUTIVE_FAILURES) low_cost_model = str(repo_policy.get("low_cost_model") or DEFAULT_LOW_COST_MODEL) quota_low_behavior = str(repo_policy.get("quota_low_behavior") or "low_cost_model").strip().lower() @@ -163,6 +168,8 @@ def decide_automation_execution( effective_mode = MODE_REVIEW_ONLY human_review_required = True reasons.append(f"repo max autonomy is {max_autonomy}") + if autonomy_config_error: + reasons.append(autonomy_config_error) if max_autonomy == AUTONOMY_MANUAL: action = EXECUTION_HUMAN_REVIEW diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 7ff0ad12..37c1cb04 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -42,7 +42,7 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") - self.assertEqual(control["action"], "continue") + self.assertEqual(control["action"], "review_only") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) @@ -91,6 +91,7 @@ def snapshot(self, limit=100): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") self.assertIsNone(ledger.requested_limit) + self.assertEqual(control["action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 35924385..b23beb48 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -93,6 +93,21 @@ def test_repo_policy_can_force_review_only(self) -> None: self.assertTrue(result["human_review_required"]) self.assertFalse(result["auto_fix_allowed"]) + def test_invalid_repo_autonomy_fails_closed(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/CryptoLivePoolPipelines", + requested_mode=MODE_REVIEW_AND_FIX, + control_action=CONTROL_CONTINUE, + service_health="healthy", + quota_status="ok", + org_health_status="ok", + policy={"repositories": {"QuantStrategyLab/CryptoLivePoolPipelines": {"max_autonomy": "auto_mrege"}}}, + ) + + self.assertEqual(result["max_autonomy"], "manual") + self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) + self.assertTrue(any("invalid max_autonomy" in reason for reason in result["reasons"])) + def test_consecutive_failures_force_human_review(self) -> None: runs = [ { @@ -123,6 +138,22 @@ def test_consecutive_failures_force_human_review(self) -> None: self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) + def test_running_state_does_not_clear_failure_streak(self) -> None: + runs = [ + { + "task_name": "monthly", + "task_state": "running", + "metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"}, + }, + { + "task_name": "monthly", + "task_state": "failed", + "metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"}, + }, + ] + + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge", task_name="monthly"), 1) + def test_invalid_failure_threshold_falls_back_safely(self) -> None: result = decide_automation_execution( repo="QuantStrategyLab/AIAuditBridge", From c1c1e7ef2d145e8cd859d350df92e38b855ac8c3 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:13:49 +0800 Subject: [PATCH 04/72] fix: fail closed automation policy loading Co-Authored-By: Codex --- docs/async_service_deployment.md | 8 +++-- scripts/deploy_codex_audit_service.sh | 23 ++++++++++++++ service/ai_gateway_service.py | 17 ++++++++-- service/automation_decision.py | 32 +++++++++++++++---- tests/test_ai_gateway_automation_control.py | 35 +++++++++++++++++++++ tests/test_automation_decision.py | 22 ++++++++++--- tests/test_run_monthly_codex_audit.py | 2 ++ 7 files changed, 124 insertions(+), 15 deletions(-) diff --git a/docs/async_service_deployment.md b/docs/async_service_deployment.md index cd36014b..ff736931 100644 --- a/docs/async_service_deployment.md +++ b/docs/async_service_deployment.md @@ -72,8 +72,9 @@ bash scripts/deploy_codex_audit_service.sh deploy The job directory should be owned by the service user and mode `0700`. The deploy script points `CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH` to -`${CODEX_AUDIT_SERVICE_JOB_DIR}/execution_policy.json`. If present, this -service-owned file can cap repo autonomy without trusting the reviewed checkout: +`${CODEX_AUDIT_SERVICE_JOB_DIR}/execution_policy.json` and creates a conservative +default file if it is missing. This service-owned file can cap repo autonomy +without trusting the reviewed checkout: ```json { @@ -91,6 +92,9 @@ service-owned file can cap repo autonomy without trusting the reviewed checkout: } ``` +If this configured file later becomes unreadable or malformed, the service +fails closed for execution decisions until the file is repaired. + The service should rely on an authenticated Codex CLI session and must not inject OpenAI/Codex API keys into the Codex subprocess. With `CODEX_AUDIT_SERVICE_CODEX_ACCOUNT_USAGE=1`, `/v1/ai/quota` includes a diff --git a/scripts/deploy_codex_audit_service.sh b/scripts/deploy_codex_audit_service.sh index 52705557..1756f655 100644 --- a/scripts/deploy_codex_audit_service.sh +++ b/scripts/deploy_codex_audit_service.sh @@ -214,6 +214,28 @@ write_admin_env_file_if_needed() { trap - RETURN } +write_default_execution_policy_if_missing() { + local owner="$1" + local policy_path="${JOB_DIR}/execution_policy.json" + if [ -e "$policy_path" ]; then + return + fi + local tmp + tmp="$(mktemp)" + cat >"$tmp" <<'EOF_POLICY' +{ + "default": { + "max_autonomy": "auto_pr", + "max_consecutive_failures": 3, + "low_cost_model": "gpt-5.4-mini" + }, + "repositories": {} +} +EOF_POLICY + sudo install -m 0600 -o "$owner" -g "$owner" "$tmp" "$policy_path" + rm -f "$tmp" +} + write_audit_service_unit() { local runner_user runner_home runner_user="$(id -un)" @@ -510,6 +532,7 @@ deploy() { install_file "scripts/codex_audit_service.py" "${DEPLOY_DIR}/scripts/codex_audit_service.py" "0755" install_service_package sudo install -d -m 0700 -o "$runner_user" -g "$runner_user" "$JOB_DIR" + write_default_execution_policy_if_missing "$runner_user" write_admin_env_file_if_needed write_audit_service_unit write_managed_audit_service_dropin diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 2618daa0..e8e19c82 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -35,6 +35,7 @@ from service.auth import authenticate from service.contracts import ( + MODE_REVIEW_AND_FIX, MODE_REVIEW_ONLY, TASK_ANALYZE, TASK_EXECUTE, @@ -508,7 +509,7 @@ def _public_job_payload(job: dict[str, Any]) -> dict[str, object]: return payload -def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mode: str = MODE_REVIEW_ONLY) -> dict[str, Any]: +def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mode: str = MODE_REVIEW_AND_FIX) -> dict[str, Any]: try: org_health = read_org_health() except Exception: @@ -520,8 +521,10 @@ def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mo control = suggest_control_action(get_health_monitor().status, quota_status, org_health) try: recent_runs = get_automation_run_ledger().snapshot(limit=None)["runs"] + ledger_unavailable = False except Exception: recent_runs = [] + ledger_unavailable = True execution = decide_automation_execution( repo=repo or "unknown", task_name=task_name, @@ -533,6 +536,14 @@ def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mo recent_runs=recent_runs, policy=load_execution_policy(), ) + if ledger_unavailable: + execution["action"] = EXECUTION_HUMAN_REVIEW + execution["effective_mode"] = MODE_REVIEW_ONLY + execution["human_review_required"] = True + execution["auto_fix_allowed"] = False + reasons = execution.get("reasons") if isinstance(execution.get("reasons"), list) else [] + reasons.append("automation ledger unavailable; forcing human review") + execution["reasons"] = reasons original_action = str(control.get("action") or CONTROL_REVIEW_ONLY) strict_action = original_action if execution.get("action") == EXECUTION_HUMAN_REVIEW: @@ -1459,7 +1470,7 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - mode = str(params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY) + mode = str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX) _json_response(self, HTTPStatus.OK, {"status": "ok", "control": _automation_control_snapshot(repo, requested_mode=mode)}) def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, Any]) -> None: @@ -1541,7 +1552,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st control = _automation_control_snapshot( repo, task_name=str(payload.get("task") or payload.get("task_name") or ""), - requested_mode=str(payload.get("mode") or MODE_REVIEW_ONLY), + requested_mode=str(payload.get("mode") or MODE_REVIEW_AND_FIX), ) metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {} ledger = get_automation_run_ledger() diff --git a/service/automation_decision.py b/service/automation_decision.py index 4c2d24fe..8b35b198 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -29,6 +29,7 @@ DEFAULT_MAX_CONSECUTIVE_FAILURES = 3 DEFAULT_LOW_COST_MODEL = "gpt-5.4-mini" EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH" +POLICY_LOAD_ERROR_KEY = "_load_error" QUOTA_STATUS_SEVERITY = { "ok": 0, "healthy": 0, @@ -83,6 +84,16 @@ def _repo_from_run(run: dict[str, Any]) -> str: return str(metadata.get("source_repository") or metadata.get("repository") or "") +def _fail_closed_policy(reason: str) -> dict[str, Any]: + return { + POLICY_LOAD_ERROR_KEY: reason, + "default": { + "max_autonomy": AUTONOMY_MANUAL, + "max_consecutive_failures": 1, + }, + } + + def load_execution_policy(path: Path | None = None) -> dict[str, Any]: """Load service-owned execution policy for repo autonomy thresholds.""" if path is None: @@ -90,11 +101,15 @@ def load_execution_policy(path: Path | None = None) -> dict[str, Any]: if not configured: return {} path = Path(configured).expanduser() + if not path.exists(): + return _fail_closed_policy("execution policy file is unavailable") try: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): - return {} - return payload if isinstance(payload, dict) else {} + return _fail_closed_policy("execution policy file is unreadable") + if not isinstance(payload, dict): + return _fail_closed_policy("execution policy file is invalid") + return payload def repo_execution_policy(repo: str, policy: dict[str, Any] | None = None) -> dict[str, Any]: @@ -146,6 +161,7 @@ def decide_automation_execution( ) -> dict[str, Any]: """Produce a safe execution decision from health, quota, failures, and repo policy.""" repo_policy = repo_execution_policy(repo, policy) + policy_load_error = str((policy or {}).get(POLICY_LOAD_ERROR_KEY) or "") if isinstance(policy, dict) else "" max_autonomy, autonomy_config_error = _parse_autonomy(repo_policy.get("max_autonomy"), AUTONOMY_AUTO_PR) max_failures = _safe_positive_int(repo_policy.get("max_consecutive_failures"), DEFAULT_MAX_CONSECUTIVE_FAILURES) low_cost_model = str(repo_policy.get("low_cost_model") or DEFAULT_LOW_COST_MODEL) @@ -170,6 +186,8 @@ def decide_automation_execution( reasons.append(f"repo max autonomy is {max_autonomy}") if autonomy_config_error: reasons.append(autonomy_config_error) + if policy_load_error: + reasons.append(policy_load_error) if max_autonomy == AUTONOMY_MANUAL: action = EXECUTION_HUMAN_REVIEW @@ -199,16 +217,18 @@ def decide_automation_execution( if quota in {"low", "constrained"}: effective_model = effective_model or low_cost_model or recommend_model(0.0) if quota_low_behavior == "defer": - action = EXECUTION_DEFER - defer = True + if action != EXECUTION_HUMAN_REVIEW: + action = EXECUTION_DEFER + defer = True effective_mode = MODE_REVIEW_ONLY human_review_required = True reasons.append(f"quota status is {quota}; deferring automation") else: reasons.append(f"quota status is {quota}; recommending low-cost model") elif quota in {"exhausted", "blocked"}: - action = EXECUTION_DEFER - defer = True + if action != EXECUTION_HUMAN_REVIEW: + action = EXECUTION_DEFER + defer = True effective_mode = MODE_REVIEW_ONLY human_review_required = True reasons.append(f"quota status is {quota}; deferring automation") diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 37c1cb04..9f2437a3 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -13,6 +13,24 @@ class TestAutomationControlSnapshot(unittest.TestCase): + def test_control_snapshot_preserves_continue_for_healthy_default_mode(self) -> None: + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={}), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") + + self.assertEqual(control["action"], "continue") + self.assertEqual(control["execution"]["effective_mode"], "review_and_fix") + self.assertTrue(control["execution"]["auto_fix_allowed"]) + def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: with TemporaryDirectory() as tmp: policy_path = Path(tmp) / "execution_policy.json" @@ -95,6 +113,23 @@ def snapshot(self, limit=100): self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) + def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: (_ for _ in ()).throw(RuntimeError("boom"))})() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={}), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") + + self.assertEqual(control["action"], "escalate") + self.assertEqual(control["execution"]["action"], "human_review") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index b23beb48..ce016c2b 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -49,11 +49,11 @@ def test_degraded_health_forces_review_only(self) -> None: self.assertFalse(result["auto_fix_allowed"]) self.assertTrue(result["human_review_required"]) - def test_exhausted_quota_defers_execution(self) -> None: + def test_exhausted_quota_defers_execution_without_stronger_guard(self) -> None: result = decide_automation_execution( repo="QuantStrategyLab/AIAuditBridge", requested_mode=MODE_REVIEW_AND_FIX, - control_action=CONTROL_ESCALATE, + control_action=CONTROL_CONTINUE, service_health="healthy", quota_status={"status": "exhausted"}, org_health_status="ok", @@ -63,6 +63,20 @@ def test_exhausted_quota_defers_execution(self) -> None: self.assertTrue(result["defer"]) self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) + def test_human_review_dominates_exhausted_quota_defer(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode=MODE_REVIEW_AND_FIX, + control_action=CONTROL_ESCALATE, + service_health="healthy", + quota_status={"status": "exhausted"}, + org_health_status="ok", + ) + + self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) + self.assertFalse(result["defer"]) + self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) + def test_low_quota_recommends_low_cost_model(self) -> None: result = decide_automation_execution( repo="QuantStrategyLab/AIAuditBridge", @@ -168,12 +182,12 @@ def test_invalid_failure_threshold_falls_back_safely(self) -> None: self.assertEqual(result["max_consecutive_failures"], 3) self.assertEqual(result["action"], EXECUTION_RUN) - def test_load_execution_policy_ignores_malformed_files(self) -> None: + def test_load_execution_policy_fails_closed_for_malformed_files(self) -> None: with TemporaryDirectory() as tmp: path = Path(tmp) / "policy.json" path.write_text("not-json", encoding="utf-8") - self.assertEqual(load_execution_policy(path), {}) + self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "manual") path.write_text(json.dumps({"default": {"max_autonomy": "review_only"}}), encoding="utf-8") self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "review_only") diff --git a/tests/test_run_monthly_codex_audit.py b/tests/test_run_monthly_codex_audit.py index 644d6748..afd1bf93 100644 --- a/tests/test_run_monthly_codex_audit.py +++ b/tests/test_run_monthly_codex_audit.py @@ -2343,6 +2343,8 @@ def test_vps_deploy_adds_nginx_audit_route_without_router_service(self) -> None: self.assertIn("location ^~ /v1/codex-audit/", deploy_script) self.assertIn("CODEX_AUDIT_SERVICE_JOB_DIR", deploy_script) self.assertIn("CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH", deploy_script) + self.assertIn("write_default_execution_policy_if_missing", deploy_script) + self.assertIn('"max_consecutive_failures": 3', deploy_script) self.assertIn("codex_pr_review.yml@refs/pull/*/merge", deploy_script) self.assertIn("refs/pull/*/merge", deploy_script) self.assertIn("QuantStrategyLab/AIAuditBridge,QuantStrategyLab/CryptoLivePoolPipelines", deploy_script) From 3f5fc852d81fc80a12981fd82a042371456ad6b0 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:20:25 +0800 Subject: [PATCH 05/72] fix: keep automation control default conservative Co-Authored-By: Codex --- service/ai_gateway_service.py | 7 +++---- service/automation_decision.py | 18 ++++++++++++++++++ tests/test_ai_gateway_automation_control.py | 20 +++++++++++++++++++- tests/test_automation_decision.py | 6 ++++++ 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index e8e19c82..8aaf43de 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -35,7 +35,6 @@ from service.auth import authenticate from service.contracts import ( - MODE_REVIEW_AND_FIX, MODE_REVIEW_ONLY, TASK_ANALYZE, TASK_EXECUTE, @@ -509,7 +508,7 @@ def _public_job_payload(job: dict[str, Any]) -> dict[str, object]: return payload -def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mode: str = MODE_REVIEW_AND_FIX) -> dict[str, Any]: +def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mode: str = MODE_REVIEW_ONLY) -> dict[str, Any]: try: org_health = read_org_health() except Exception: @@ -1470,7 +1469,7 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - mode = str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX) + mode = str(params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY) _json_response(self, HTTPStatus.OK, {"status": "ok", "control": _automation_control_snapshot(repo, requested_mode=mode)}) def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, Any]) -> None: @@ -1552,7 +1551,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st control = _automation_control_snapshot( repo, task_name=str(payload.get("task") or payload.get("task_name") or ""), - requested_mode=str(payload.get("mode") or MODE_REVIEW_AND_FIX), + requested_mode=str(payload.get("mode") or MODE_REVIEW_ONLY), ) metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {} ledger = get_automation_run_ledger() diff --git a/service/automation_decision.py b/service/automation_decision.py index 8b35b198..912532f3 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -94,6 +94,21 @@ def _fail_closed_policy(reason: str) -> dict[str, Any]: } +def _validate_execution_policy(payload: dict[str, Any]) -> str: + default_policy = payload.get("default") + if default_policy is not None and not isinstance(default_policy, dict): + return "execution policy default section is invalid" + repositories = payload.get("repositories") + if repositories is None: + return "" + if not isinstance(repositories, dict): + return "execution policy repositories section is invalid" + for repo, repo_policy in repositories.items(): + if not isinstance(repo_policy, dict): + return f"execution policy override for {repo!r} is invalid" + return "" + + def load_execution_policy(path: Path | None = None) -> dict[str, Any]: """Load service-owned execution policy for repo autonomy thresholds.""" if path is None: @@ -109,6 +124,9 @@ def load_execution_policy(path: Path | None = None) -> dict[str, Any]: return _fail_closed_policy("execution policy file is unreadable") if not isinstance(payload, dict): return _fail_closed_policy("execution policy file is invalid") + schema_error = _validate_execution_policy(payload) + if schema_error: + return _fail_closed_policy(schema_error) return payload diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 9f2437a3..6c773fe2 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -13,7 +13,7 @@ class TestAutomationControlSnapshot(unittest.TestCase): - def test_control_snapshot_preserves_continue_for_healthy_default_mode(self) -> None: + def test_control_snapshot_defaults_to_review_only_for_healthy_repo(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -27,6 +27,24 @@ def test_control_snapshot_preserves_continue_for_healthy_default_mode(self) -> N ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") + self.assertEqual(control["action"], "review_only") + self.assertEqual(control["execution"]["effective_mode"], "review_only") + self.assertFalse(control["execution"]["auto_fix_allowed"]) + + def test_control_snapshot_preserves_continue_for_explicit_review_and_fix_mode(self) -> None: + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={}), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") + self.assertEqual(control["action"], "continue") self.assertEqual(control["execution"]["effective_mode"], "review_and_fix") self.assertTrue(control["execution"]["auto_fix_allowed"]) diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index ce016c2b..7afca9c1 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -189,6 +189,12 @@ def test_load_execution_policy_fails_closed_for_malformed_files(self) -> None: self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "manual") + path.write_text(json.dumps({"default": []}), encoding="utf-8") + self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "manual") + + path.write_text(json.dumps({"repositories": {"QuantStrategyLab/AIAuditBridge": "bad"}}), encoding="utf-8") + self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "manual") + path.write_text(json.dumps({"default": {"max_autonomy": "review_only"}}), encoding="utf-8") self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "review_only") From 6ba9b3f30bf92c2023a387f7800324b1577ed6b4 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:29:07 +0800 Subject: [PATCH 06/72] fix: include pending runs in automation decisions Co-Authored-By: Codex --- service/ai_gateway_service.py | 81 ++++++++++++++------- tests/test_ai_gateway_automation_control.py | 39 +++++++++- 2 files changed, 90 insertions(+), 30 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 8aaf43de..c676f924 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -35,6 +35,7 @@ from service.auth import authenticate from service.contracts import ( + MODE_REVIEW_AND_FIX, MODE_REVIEW_ONLY, TASK_ANALYZE, TASK_EXECUTE, @@ -508,7 +509,13 @@ def _public_job_payload(job: dict[str, Any]) -> dict[str, object]: return payload -def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mode: str = MODE_REVIEW_ONLY) -> dict[str, Any]: +def _automation_control_snapshot( + repo: str, + *, + task_name: str = "", + requested_mode: str = MODE_REVIEW_ONLY, + pending_run: dict[str, Any] | None = None, +) -> dict[str, Any]: try: org_health = read_org_health() except Exception: @@ -524,6 +531,8 @@ def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mo except Exception: recent_runs = [] ledger_unavailable = True + if pending_run is not None: + recent_runs = [pending_run, *recent_runs] execution = decide_automation_execution( repo=repo or "unknown", task_name=task_name, @@ -549,7 +558,11 @@ def _automation_control_snapshot(repo: str, *, task_name: str = "", requested_mo strict_action = CONTROL_ESCALATE elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX - elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: + elif ( + execution.get("requested_mode") == MODE_REVIEW_AND_FIX + and execution.get("effective_mode") == MODE_REVIEW_ONLY + and strict_action == CONTROL_CONTINUE + ): strict_action = CONTROL_REVIEW_ONLY if strict_action != original_action: control["action"] = strict_action @@ -691,24 +704,32 @@ def _automation_triage_snapshot( def _record_job_automation_run(job: dict[str, Any]) -> None: try: repo = str(job.get("source_repository") or job.get("repository") or "unknown") - control = _automation_control_snapshot(repo, task_name=str(job.get("task") or ""), requested_mode=str(job.get("mode") or MODE_REVIEW_ONLY)) + task_name = str(job.get("task") or "") + task_state = job_task_state(job) + metadata = { + "origin": "service_job", + "repository": repo, + "source_repository": str(job.get("source_repository") or ""), + "caller_repository": str(job.get("repository") or ""), + "source_ref": str(job.get("source_ref") or ""), + "mode": str(job.get("mode") or ""), + "failure_category": str(job.get("failure_category") or ""), + } + control = _automation_control_snapshot( + repo, + task_name=task_name, + requested_mode=str(job.get("mode") or MODE_REVIEW_ONLY), + pending_run={"task_name": task_name, "task_state": task_state, "metadata": metadata}, + ) get_automation_run_ledger().record( str(job.get("job_id") or ""), - job_task_state(job), - task_name=str(job.get("task") or ""), + task_state, + task_name=task_name, suggested_action=str(control.get("action") or ""), service_health=str(control.get("service_health") or ""), quota_status=str(control.get("quota_status") or ""), org_health_status=str(control.get("org_health_status") or ""), - metadata={ - "origin": "service_job", - "repository": repo, - "source_repository": str(job.get("source_repository") or ""), - "caller_repository": str(job.get("repository") or ""), - "source_ref": str(job.get("source_ref") or ""), - "mode": str(job.get("mode") or ""), - "failure_category": str(job.get("failure_category") or ""), - }, + metadata=metadata, owner_repository=repo, ) except Exception as exc: @@ -1548,11 +1569,6 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st _validate_source_repo_org(claims, source_repo) _assert_source_repository_owner_or_operator(claims, source_repo) repo = source_repo or str(claims.get("repository") or "unknown") - control = _automation_control_snapshot( - repo, - task_name=str(payload.get("task") or payload.get("task_name") or ""), - requested_mode=str(payload.get("mode") or MODE_REVIEW_ONLY), - ) metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {} ledger = get_automation_run_ledger() run_id = str(payload.get("run_id") or payload.get("job_id") or "") @@ -1567,21 +1583,30 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st if existing_metadata.get("origin") == "service_job": raise PermissionError("automation run is service-owned") _assert_automation_run_access(existing, claims) + task_name = str(payload.get("task") or payload.get("task_name") or "") + task_state = str(payload.get("task_state") or payload.get("state") or "running") + run_metadata = { + **metadata, + "origin": "external_workflow", + "repository": repo, + "source_repository": source_repo, + "caller_repository": str(claims.get("repository") or ""), + } + control = _automation_control_snapshot( + repo, + task_name=task_name, + requested_mode=str(payload.get("mode") or MODE_REVIEW_ONLY), + pending_run={"task_name": task_name, "task_state": task_state, "metadata": run_metadata}, + ) record = get_automation_run_ledger().record( run_id, - str(payload.get("task_state") or payload.get("state") or "running"), - task_name=str(payload.get("task") or payload.get("task_name") or ""), + task_state, + task_name=task_name, suggested_action=str(control.get("action") or ""), service_health=str(control.get("service_health") or ""), quota_status=control.get("quota_status") or "", org_health_status=str(control.get("org_health_status") or ""), - metadata={ - **metadata, - "origin": "external_workflow", - "repository": repo, - "source_repository": source_repo, - "caller_repository": str(claims.get("repository") or ""), - }, + metadata=run_metadata, owner_repository=repo, ) _json_response(self, HTTPStatus.OK, {"status": "ok", "run": record, "control": control}) diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 6c773fe2..2c73516b 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -27,7 +27,7 @@ def test_control_snapshot_defaults_to_review_only_for_healthy_repo(self) -> None ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") - self.assertEqual(control["action"], "review_only") + self.assertEqual(control["action"], "continue") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) @@ -76,7 +76,7 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: patch("service.ai_gateway_service.get_quota_manager", return_value=quota), patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), ): - control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") self.assertEqual(control["action"], "review_only") self.assertEqual(control["execution"]["effective_mode"], "review_only") @@ -131,6 +131,41 @@ def snapshot(self, limit=100): self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) + def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None: + runs = [ + { + "task_name": "monthly", + "task_state": "failed", + "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + } + ] + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": runs}})() + pending_run = { + "task_name": "monthly", + "task_state": "failed", + "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + } + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={"default": {"max_consecutive_failures": 2}}), + ): + control = _automation_control_snapshot( + "QuantStrategyLab/TargetRepo", + task_name="monthly", + requested_mode="review_and_fix", + pending_run=pending_run, + ) + + self.assertEqual(control["action"], "escalate") + self.assertEqual(control["execution"]["action"], "human_review") + self.assertEqual(control["execution"]["consecutive_failures"], 2) + def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() From 0848dd8b0b1e0627bc0359a55515e856937daf72 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:35:35 +0800 Subject: [PATCH 07/72] fix: strengthen defer and pending-run guards Co-Authored-By: Codex --- service/ai_gateway_service.py | 19 +++++-- service/automation_run_ledger.py | 1 + tests/test_ai_gateway_automation_control.py | 58 +++++++++++++++++++++ tests/test_automation_run_ledger.py | 11 ++++ 4 files changed, 86 insertions(+), 3 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index c676f924..8ccc9bcd 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -532,6 +532,9 @@ def _automation_control_snapshot( recent_runs = [] ledger_unavailable = True if pending_run is not None: + pending_run_id = str(pending_run.get("run_id") or "") + if pending_run_id: + recent_runs = [run for run in recent_runs if str(run.get("run_id") or "") != pending_run_id] recent_runs = [pending_run, *recent_runs] execution = decide_automation_execution( repo=repo or "unknown", @@ -557,7 +560,7 @@ def _automation_control_snapshot( if execution.get("action") == EXECUTION_HUMAN_REVIEW: strict_action = CONTROL_ESCALATE elif execution.get("action") == EXECUTION_DEFER: - strict_action = CONTROL_PAUSE_AUTO_FIX + strict_action = CONTROL_ESCALATE elif ( execution.get("requested_mode") == MODE_REVIEW_AND_FIX and execution.get("effective_mode") == MODE_REVIEW_ONLY @@ -719,7 +722,12 @@ def _record_job_automation_run(job: dict[str, Any]) -> None: repo, task_name=task_name, requested_mode=str(job.get("mode") or MODE_REVIEW_ONLY), - pending_run={"task_name": task_name, "task_state": task_state, "metadata": metadata}, + pending_run={ + "run_id": str(job.get("job_id") or ""), + "task_name": task_name, + "task_state": task_state, + "metadata": metadata, + }, ) get_automation_run_ledger().record( str(job.get("job_id") or ""), @@ -1596,7 +1604,12 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st repo, task_name=task_name, requested_mode=str(payload.get("mode") or MODE_REVIEW_ONLY), - pending_run={"task_name": task_name, "task_state": task_state, "metadata": run_metadata}, + pending_run={ + "run_id": run_id, + "task_name": task_name, + "task_state": task_state, + "metadata": run_metadata, + }, ) record = get_automation_run_ledger().record( run_id, diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index c62637f6..4d1827b6 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -467,6 +467,7 @@ def get(self, run_id: str) -> dict[str, Any] | None: return self._public_entry(entry) if entry else None def snapshot(self, *, limit: int | None = 100, include_events: bool = False) -> dict[str, Any]: + """Return retained runs; ``limit=None`` returns the full retained ledger.""" with self._lock: self._refresh_from_disk_locked() retained_runs = [ diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 2c73516b..fe028b6c 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -134,6 +134,7 @@ def snapshot(self, limit=100): def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None: runs = [ { + "run_id": "previous-run", "task_name": "monthly", "task_state": "failed", "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, @@ -143,6 +144,7 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": runs}})() pending_run = { + "run_id": "current-run", "task_name": "monthly", "task_state": "failed", "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, @@ -166,6 +168,62 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) + def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: + runs = [ + { + "run_id": "current-run", + "task_name": "monthly", + "task_state": "failed", + "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + } + ] + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": runs}})() + pending_run = { + "run_id": "current-run", + "task_name": "monthly", + "task_state": "failed", + "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + } + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={"default": {"max_consecutive_failures": 2}}), + ): + control = _automation_control_snapshot( + "QuantStrategyLab/TargetRepo", + task_name="monthly", + requested_mode="review_and_fix", + pending_run=pending_run, + ) + + self.assertEqual(control["action"], "continue") + self.assertEqual(control["execution"]["consecutive_failures"], 1) + + def test_control_snapshot_maps_defer_to_legacy_escalate(self) -> None: + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "low"}})() + ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch( + "service.ai_gateway_service.load_execution_policy", + return_value={"default": {"quota_low_behavior": "defer"}}, + ), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") + + self.assertEqual(control["action"], "escalate") + self.assertEqual(control["execution"]["action"], "defer") + def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index d560fd9f..e4ca5655 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -145,6 +145,17 @@ def test_snapshot_summarizes_terminal_and_active_runs(self) -> None: self.assertEqual(snapshot["summary"]["suggested_actions"][CONTROL_CONTINUE], 2) self.assertNotIn("events", snapshot["runs"][0]) + def test_snapshot_none_limit_returns_all_retained_runs(self) -> None: + self.ledger.record("run-1", "running") + self.ledger.record("run-2", "running") + self.ledger.record("run-3", "running") + + snapshot = self.ledger.snapshot(limit=None) + + self.assertEqual(snapshot["summary"]["total_runs"], 3) + self.assertEqual(snapshot["summary"]["returned_runs"], 3) + self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-1", "run-2", "run-3"}) + def test_snapshot_can_include_bounded_history(self) -> None: ledger = AutomationRunLedger(max_events_per_run=2) ledger.record("run-1", "queued") From 59e0ee3cecec22344b6794831b4a89d2cec75203 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:41:37 +0800 Subject: [PATCH 08/72] fix: normalize execution policy decisions Co-Authored-By: Codex --- docs/async_service_deployment.md | 5 +++-- service/automation_decision.py | 18 +++++++++++++---- tests/test_automation_decision.py | 33 ++++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/docs/async_service_deployment.md b/docs/async_service_deployment.md index ff736931..e61bc461 100644 --- a/docs/async_service_deployment.md +++ b/docs/async_service_deployment.md @@ -92,8 +92,9 @@ without trusting the reviewed checkout: } ``` -If this configured file later becomes unreadable or malformed, the service -fails closed for execution decisions until the file is repaired. +If this configured file is missing, unreadable, malformed, or the policy path is +not configured, the service fails closed for execution decisions until the +configuration is repaired. The service should rely on an authenticated Codex CLI session and must not inject OpenAI/Codex API keys into the Codex subprocess. diff --git a/service/automation_decision.py b/service/automation_decision.py index 912532f3..fe3369f0 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -62,6 +62,10 @@ def _normalize_mode(value: str) -> str: return MODE_REVIEW_AND_FIX if str(value or "").strip() == MODE_REVIEW_AND_FIX else MODE_REVIEW_ONLY +def _normalize_repo_id(value: Any) -> str: + return str(value or "").strip().lower() + + def _parse_autonomy(value: Any, default: str = AUTONOMY_AUTO_PR) -> tuple[str, str]: level = str(value or "").strip().lower() if not level: @@ -114,7 +118,7 @@ def load_execution_policy(path: Path | None = None) -> dict[str, Any]: if path is None: configured = os.environ.get(EXECUTION_POLICY_PATH_ENV, "").strip() if not configured: - return {} + return _fail_closed_policy("execution policy path is not configured") path = Path(configured).expanduser() if not path.exists(): return _fail_closed_policy("execution policy file is unavailable") @@ -135,7 +139,12 @@ def repo_execution_policy(repo: str, policy: dict[str, Any] | None = None) -> di raw = policy if isinstance(policy, dict) else {} defaults = raw.get("default") if isinstance(raw.get("default"), dict) else {} repositories = raw.get("repositories") if isinstance(raw.get("repositories"), dict) else {} - override = repositories.get(repo) if isinstance(repositories.get(repo), dict) else {} + normalized_repo = _normalize_repo_id(repo) + override = {} + for configured_repo, configured_policy in repositories.items(): + if _normalize_repo_id(configured_repo) == normalized_repo and isinstance(configured_policy, dict): + override = configured_policy + break return {**defaults, **override} @@ -147,10 +156,11 @@ def consecutive_failure_count( ) -> int: """Count latest consecutive failed runs for one repo/task from newest-first runs.""" count = 0 + normalized_repo = _normalize_repo_id(repo) for run in runs: if not isinstance(run, dict): continue - if repo and _repo_from_run(run) != repo: + if normalized_repo and _normalize_repo_id(_repo_from_run(run)) != normalized_repo: continue if task_name and str(run.get("task_name") or "") != task_name: continue @@ -233,7 +243,7 @@ def decide_automation_execution( reasons.append("health unhealthy; forcing human review") if quota in {"low", "constrained"}: - effective_model = effective_model or low_cost_model or recommend_model(0.0) + effective_model = low_cost_model or recommend_model(0.0) if quota_low_behavior == "defer": if action != EXECUTION_HUMAN_REVIEW: action = EXECUTION_DEFER diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 7afca9c1..1e450e56 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -3,9 +3,11 @@ from __future__ import annotations import json +import os from pathlib import Path from tempfile import TemporaryDirectory import unittest +from unittest.mock import patch from service.automation_decision import ( EXECUTION_DEFER, @@ -92,9 +94,23 @@ def test_low_quota_recommends_low_cost_model(self) -> None: self.assertEqual(result["effective_model"], "gpt-5.4-mini") self.assertTrue(any("low-cost model" in reason for reason in result["reasons"])) + def test_low_quota_overrides_requested_expensive_model(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode=MODE_REVIEW_AND_FIX, + requested_model="gpt-5.4-pro", + control_action=CONTROL_CONTINUE, + service_health="healthy", + quota_status="low", + org_health_status="ok", + policy={"default": {"low_cost_model": "gpt-5.4-mini"}}, + ) + + self.assertEqual(result["effective_model"], "gpt-5.4-mini") + def test_repo_policy_can_force_review_only(self) -> None: result = decide_automation_execution( - repo="QuantStrategyLab/CryptoLivePoolPipelines", + repo="quantstrategylab/cryptolivepoolpipelines", requested_mode=MODE_REVIEW_AND_FIX, control_action=CONTROL_CONTINUE, service_health="healthy", @@ -107,6 +123,17 @@ def test_repo_policy_can_force_review_only(self) -> None: self.assertTrue(result["human_review_required"]) self.assertFalse(result["auto_fix_allowed"]) + def test_failure_streak_matching_is_case_insensitive(self) -> None: + runs = [ + { + "task_name": "monthly", + "task_state": "failed", + "metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"}, + }, + ] + + self.assertEqual(consecutive_failure_count(runs, repo="quantstrategylab/aiauditbridge", task_name="monthly"), 1) + def test_invalid_repo_autonomy_fails_closed(self) -> None: result = decide_automation_execution( repo="QuantStrategyLab/CryptoLivePoolPipelines", @@ -198,6 +225,10 @@ def test_load_execution_policy_fails_closed_for_malformed_files(self) -> None: path.write_text(json.dumps({"default": {"max_autonomy": "review_only"}}), encoding="utf-8") self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "review_only") + def test_load_execution_policy_fails_closed_when_path_unset(self) -> None: + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(load_execution_policy()["default"]["max_autonomy"], "manual") + if __name__ == "__main__": unittest.main() From 0cc9ab96e4707315c80ff990715a14966863e268 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:47:23 +0800 Subject: [PATCH 09/72] fix: enforce repo-level trusted failure streaks Co-Authored-By: Codex --- service/automation_decision.py | 8 ++++--- tests/test_ai_gateway_automation_control.py | 14 ++++++------- tests/test_automation_decision.py | 23 +++++++++++++++------ 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/service/automation_decision.py b/service/automation_decision.py index fe3369f0..b708beb3 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -30,6 +30,7 @@ DEFAULT_LOW_COST_MODEL = "gpt-5.4-mini" EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH" POLICY_LOAD_ERROR_KEY = "_load_error" +TRUSTED_FAILURE_ORIGINS = frozenset({"service_job"}) QUOTA_STATUS_SEVERITY = { "ok": 0, "healthy": 0, @@ -154,15 +155,16 @@ def consecutive_failure_count( repo: str, task_name: str = "", ) -> int: - """Count latest consecutive failed runs for one repo/task from newest-first runs.""" + """Count latest consecutive trusted failed runs for one repo from newest-first runs.""" count = 0 normalized_repo = _normalize_repo_id(repo) for run in runs: if not isinstance(run, dict): continue - if normalized_repo and _normalize_repo_id(_repo_from_run(run)) != normalized_repo: + metadata = run.get("metadata") if isinstance(run.get("metadata"), dict) else {} + if str(metadata.get("origin") or "") not in TRUSTED_FAILURE_ORIGINS: continue - if task_name and str(run.get("task_name") or "") != task_name: + if normalized_repo and _normalize_repo_id(_repo_from_run(run)) != normalized_repo: continue state = str(run.get("task_state") or "").strip().lower() if state == "failed": diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index fe028b6c..5343fe05 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -87,7 +87,7 @@ def test_control_snapshot_scans_full_retained_ledger_for_repo_failure_streak(sel { "task_name": f"other-{index}", "task_state": "merged", - "metadata": {"source_repository": f"QuantStrategyLab/Other{index}"}, + "metadata": {"origin": "service_job", "source_repository": f"QuantStrategyLab/Other{index}"}, } for index in range(20) ] @@ -96,12 +96,12 @@ def test_control_snapshot_scans_full_retained_ledger_for_repo_failure_streak(sel { "task_name": "monthly", "task_state": "failed", - "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}, }, { "task_name": "monthly", "task_state": "failed", - "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}, }, ] ) @@ -137,7 +137,7 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None "run_id": "previous-run", "task_name": "monthly", "task_state": "failed", - "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}, } ] health = type("Health", (), {"status": "healthy"})() @@ -147,7 +147,7 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None "run_id": "current-run", "task_name": "monthly", "task_state": "failed", - "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}, } with ( @@ -174,7 +174,7 @@ def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: "run_id": "current-run", "task_name": "monthly", "task_state": "failed", - "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}, } ] health = type("Health", (), {"status": "healthy"})() @@ -184,7 +184,7 @@ def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: "run_id": "current-run", "task_name": "monthly", "task_state": "failed", - "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}, + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}, } with ( diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 1e450e56..de9877e3 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -128,12 +128,23 @@ def test_failure_streak_matching_is_case_insensitive(self) -> None: { "task_name": "monthly", "task_state": "failed", - "metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"}, + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, }, ] self.assertEqual(consecutive_failure_count(runs, repo="quantstrategylab/aiauditbridge", task_name="monthly"), 1) + def test_external_workflow_failures_do_not_force_repo_failure_streak(self) -> None: + runs = [ + { + "task_name": "monthly", + "task_state": "failed", + "metadata": {"origin": "external_workflow", "source_repository": "QuantStrategyLab/AIAuditBridge"}, + }, + ] + + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 0) + def test_invalid_repo_autonomy_fails_closed(self) -> None: result = decide_automation_execution( repo="QuantStrategyLab/CryptoLivePoolPipelines", @@ -154,12 +165,12 @@ def test_consecutive_failures_force_human_review(self) -> None: { "task_name": "monthly", "task_state": "failed", - "metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"}, + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, }, { - "task_name": "monthly", + "task_name": "runtime-health", "task_state": "failed", - "metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"}, + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, }, ] @@ -184,12 +195,12 @@ def test_running_state_does_not_clear_failure_streak(self) -> None: { "task_name": "monthly", "task_state": "running", - "metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"}, + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, }, { "task_name": "monthly", "task_state": "failed", - "metadata": {"source_repository": "QuantStrategyLab/AIAuditBridge"}, + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, }, ] From a0de099fad0b102e00deb4cfc4ddf8f45708a1f6 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 03:53:51 +0800 Subject: [PATCH 10/72] fix: validate execution policy schema Co-Authored-By: Codex --- docs/async_service_deployment.md | 3 +- scripts/deploy_codex_audit_service.sh | 3 +- service/automation_decision.py | 44 +++++++++++++++++++-- tests/test_ai_gateway_automation_control.py | 6 +++ tests/test_automation_decision.py | 23 ++++++++++- 5 files changed, 73 insertions(+), 6 deletions(-) diff --git a/docs/async_service_deployment.md b/docs/async_service_deployment.md index e61bc461..06eb6932 100644 --- a/docs/async_service_deployment.md +++ b/docs/async_service_deployment.md @@ -81,7 +81,8 @@ without trusting the reviewed checkout: "default": { "max_autonomy": "auto_pr", "max_consecutive_failures": 3, - "low_cost_model": "gpt-5.4-mini" + "low_cost_model": "gpt-5.4-mini", + "low_cost_provider": "openai" }, "repositories": { "QuantStrategyLab/CryptoLivePoolPipelines": { diff --git a/scripts/deploy_codex_audit_service.sh b/scripts/deploy_codex_audit_service.sh index 1756f655..47bc691c 100644 --- a/scripts/deploy_codex_audit_service.sh +++ b/scripts/deploy_codex_audit_service.sh @@ -227,7 +227,8 @@ write_default_execution_policy_if_missing() { "default": { "max_autonomy": "auto_pr", "max_consecutive_failures": 3, - "low_cost_model": "gpt-5.4-mini" + "low_cost_model": "gpt-5.4-mini", + "low_cost_provider": "openai" }, "repositories": {} } diff --git a/service/automation_decision.py b/service/automation_decision.py index b708beb3..28cc66c1 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -28,9 +28,14 @@ DEFAULT_MAX_CONSECUTIVE_FAILURES = 3 DEFAULT_LOW_COST_MODEL = "gpt-5.4-mini" +DEFAULT_LOW_COST_PROVIDER = "openai" EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH" POLICY_LOAD_ERROR_KEY = "_load_error" TRUSTED_FAILURE_ORIGINS = frozenset({"service_job"}) +POLICY_ALLOWED_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider", "quota_low_behavior"}) +POLICY_REQUIRED_DEFAULT_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider"}) +POLICY_LOW_QUOTA_BEHAVIORS = frozenset({"low_cost_model", "defer"}) +POLICY_PROVIDERS = frozenset({"auto", "openai", "anthropic", "api", "codex"}) QUOTA_STATUS_SEVERITY = { "ok": 0, "healthy": 0, @@ -101,16 +106,47 @@ def _fail_closed_policy(reason: str) -> dict[str, Any]: def _validate_execution_policy(payload: dict[str, Any]) -> str: default_policy = payload.get("default") - if default_policy is not None and not isinstance(default_policy, dict): + if not isinstance(default_policy, dict): return "execution policy default section is invalid" + default_error = _validate_policy_section(default_policy, section_name="default", require_defaults=True) + if default_error: + return default_error repositories = payload.get("repositories") - if repositories is None: - return "" if not isinstance(repositories, dict): return "execution policy repositories section is invalid" for repo, repo_policy in repositories.items(): if not isinstance(repo_policy, dict): return f"execution policy override for {repo!r} is invalid" + if not repo_policy: + return f"execution policy override for {repo!r} is empty" + repo_error = _validate_policy_section(repo_policy, section_name=f"override for {repo!r}", require_defaults=False) + if repo_error: + return repo_error + return "" + + +def _validate_policy_section(section: dict[str, Any], *, section_name: str, require_defaults: bool) -> str: + unknown_keys = set(section) - POLICY_ALLOWED_KEYS + if unknown_keys: + return f"execution policy {section_name} has unknown keys" + if require_defaults: + missing_keys = POLICY_REQUIRED_DEFAULT_KEYS - set(section) + if missing_keys: + return f"execution policy {section_name} is missing required keys" + if "max_autonomy" in section and str(section["max_autonomy"] or "").strip().lower() not in AUTONOMY_RANK: + return f"execution policy {section_name} has invalid max_autonomy" + if "max_consecutive_failures" in section and _safe_positive_int(section["max_consecutive_failures"], 0) <= 0: + return f"execution policy {section_name} has invalid max_consecutive_failures" + if "low_cost_model" in section and not str(section["low_cost_model"] or "").strip(): + return f"execution policy {section_name} has invalid low_cost_model" + if "low_cost_provider" in section: + provider = str(section["low_cost_provider"] or "").strip().lower() + if provider not in POLICY_PROVIDERS: + return f"execution policy {section_name} has invalid low_cost_provider" + if "quota_low_behavior" in section: + behavior = str(section["quota_low_behavior"] or "").strip().lower() + if behavior not in POLICY_LOW_QUOTA_BEHAVIORS: + return f"execution policy {section_name} has invalid quota_low_behavior" return "" @@ -195,6 +231,7 @@ def decide_automation_execution( max_autonomy, autonomy_config_error = _parse_autonomy(repo_policy.get("max_autonomy"), AUTONOMY_AUTO_PR) max_failures = _safe_positive_int(repo_policy.get("max_consecutive_failures"), DEFAULT_MAX_CONSECUTIVE_FAILURES) low_cost_model = str(repo_policy.get("low_cost_model") or DEFAULT_LOW_COST_MODEL) + low_cost_provider = str(repo_policy.get("low_cost_provider") or DEFAULT_LOW_COST_PROVIDER).strip().lower() quota_low_behavior = str(repo_policy.get("quota_low_behavior") or "low_cost_model").strip().lower() effective_mode = _normalize_mode(requested_mode) @@ -246,6 +283,7 @@ def decide_automation_execution( if quota in {"low", "constrained"}: effective_model = low_cost_model or recommend_model(0.0) + effective_provider = low_cost_provider or "auto" if quota_low_behavior == "defer": if action != EXECUTION_HUMAN_REVIEW: action = EXECUTION_DEFER diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 5343fe05..54eae460 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -55,6 +55,12 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: policy_path.write_text( json.dumps( { + "default": { + "max_autonomy": "auto_pr", + "max_consecutive_failures": 3, + "low_cost_model": "gpt-5.4-mini", + "low_cost_provider": "openai", + }, "repositories": { "QuantStrategyLab/TargetRepo": { "max_autonomy": "review_only", diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index de9877e3..013c11e2 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -91,6 +91,7 @@ def test_low_quota_recommends_low_cost_model(self) -> None: ) self.assertEqual(result["action"], EXECUTION_RUN) + self.assertEqual(result["effective_provider"], "openai") self.assertEqual(result["effective_model"], "gpt-5.4-mini") self.assertTrue(any("low-cost model" in reason for reason in result["reasons"])) @@ -106,6 +107,7 @@ def test_low_quota_overrides_requested_expensive_model(self) -> None: policy={"default": {"low_cost_model": "gpt-5.4-mini"}}, ) + self.assertEqual(result["effective_provider"], "openai") self.assertEqual(result["effective_model"], "gpt-5.4-mini") def test_repo_policy_can_force_review_only(self) -> None: @@ -233,7 +235,26 @@ def test_load_execution_policy_fails_closed_for_malformed_files(self) -> None: path.write_text(json.dumps({"repositories": {"QuantStrategyLab/AIAuditBridge": "bad"}}), encoding="utf-8") self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "manual") - path.write_text(json.dumps({"default": {"max_autonomy": "review_only"}}), encoding="utf-8") + path.write_text(json.dumps({"default": {"max_consecutive_failures": "oops"}, "repositories": {}}), encoding="utf-8") + self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "manual") + + path.write_text(json.dumps({"default": {"max_autnomy": "review_only"}, "repositories": {}}), encoding="utf-8") + self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "manual") + + path.write_text( + json.dumps( + { + "default": { + "max_autonomy": "review_only", + "max_consecutive_failures": 3, + "low_cost_model": "gpt-5.4-mini", + "low_cost_provider": "openai", + }, + "repositories": {}, + } + ), + encoding="utf-8", + ) self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "review_only") def test_load_execution_policy_fails_closed_when_path_unset(self) -> None: From acdd960e58b6beb3573eb885910c0163e4cf1569 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:00:21 +0800 Subject: [PATCH 11/72] fix: preserve non-human defer control action Co-Authored-By: Codex --- docs/ai_autonomy_architecture.md | 2 +- service/ai_gateway_service.py | 5 +++-- service/automation_decision.py | 3 +-- service/automation_run_ledger.py | 2 ++ tests/test_ai_gateway_automation_control.py | 5 +++-- tests/test_ai_gateway_service_get_routes.py | 2 +- tests/test_automation_decision.py | 6 +++--- 7 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/ai_autonomy_architecture.md b/docs/ai_autonomy_architecture.md index 0d2bd615..f786c986 100644 --- a/docs/ai_autonomy_architecture.md +++ b/docs/ai_autonomy_architecture.md @@ -342,7 +342,7 @@ AIAuditBridge 是 QuantStrategyLab 的 AI 审计控制面，负责： - `/v1/ai/automation/control` 输出 `execution` 决策快照； - health degraded / runtime pause 时，执行模式降级到 `review_only`； - quota low 时给出低成本模型建议，quota exhausted / blocked 时建议 defer； -- 连续失败达到 repo 阈值时强制 human review； +- service-owned run 的 repo-wide 连续失败达到 repo 阈值时强制 human review； - repo 级 `max_autonomy` / `max_consecutive_failures` 从服务端受控 policy 文件读取，不信任被审仓库 checkout； - 该阶段只影响调度建议和控制面输出，不自动放宽 merge / deploy 权限。 diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 8ccc9bcd..ddc7846d 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -67,6 +67,7 @@ ) from service.automation_run_ledger import ( CONTROL_CONTINUE, + CONTROL_DEFER, CONTROL_ESCALATE, CONTROL_PAUSE_AUTO_FIX, CONTROL_REVIEW_ONLY, @@ -560,7 +561,7 @@ def _automation_control_snapshot( if execution.get("action") == EXECUTION_HUMAN_REVIEW: strict_action = CONTROL_ESCALATE elif execution.get("action") == EXECUTION_DEFER: - strict_action = CONTROL_ESCALATE + strict_action = CONTROL_DEFER elif ( execution.get("requested_mode") == MODE_REVIEW_AND_FIX and execution.get("effective_mode") == MODE_REVIEW_ONLY @@ -569,8 +570,8 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY if strict_action != original_action: control["action"] = strict_action - control["requires_human_review"] = True control["auto_fix_allowed"] = False + control["requires_human_review"] = execution.get("action") != EXECUTION_DEFER reasons = control.get("reasons") if isinstance(control.get("reasons"), list) else [] reasons.append("capped by execution decision") control["reasons"] = reasons diff --git a/service/automation_decision.py b/service/automation_decision.py index 28cc66c1..c199b981 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -189,7 +189,6 @@ def consecutive_failure_count( runs: list[dict[str, Any]], *, repo: str, - task_name: str = "", ) -> int: """Count latest consecutive trusted failed runs for one repo from newest-first runs.""" count = 0 @@ -245,7 +244,7 @@ def decide_automation_execution( service = _normalize_status(service_health) quota = _normalize_quota_status(quota_status) org_health = _normalize_status(org_health_status) - failures = consecutive_failure_count(recent_runs or [], repo=repo, task_name=task_name) + failures = consecutive_failure_count(recent_runs or [], repo=repo) if AUTONOMY_RANK[max_autonomy] <= AUTONOMY_RANK[AUTONOMY_REVIEW_ONLY]: effective_mode = MODE_REVIEW_ONLY diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 4d1827b6..b778208f 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -22,6 +22,7 @@ CONTROL_REVIEW_ONLY = "review_only" CONTROL_PAUSE_AUTO_FIX = "pause_auto_fix" CONTROL_ESCALATE = "escalate" +CONTROL_DEFER = "defer" CONTROL_ACTIONS = frozenset( { @@ -29,6 +30,7 @@ CONTROL_REVIEW_ONLY, CONTROL_PAUSE_AUTO_FIX, CONTROL_ESCALATE, + CONTROL_DEFER, } ) diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 54eae460..1c05bba6 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -210,7 +210,7 @@ def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: self.assertEqual(control["action"], "continue") self.assertEqual(control["execution"]["consecutive_failures"], 1) - def test_control_snapshot_maps_defer_to_legacy_escalate(self) -> None: + def test_control_snapshot_maps_defer_to_legacy_defer(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "low"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -227,7 +227,8 @@ def test_control_snapshot_maps_defer_to_legacy_escalate(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") - self.assertEqual(control["action"], "escalate") + self.assertEqual(control["action"], "defer") + self.assertFalse(control["requires_human_review"]) self.assertEqual(control["execution"]["action"], "defer") def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index c1b6ea36..bc0ef1d4 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -580,7 +580,7 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: with urllib.request.urlopen(f"{base_url}/v1/ai/automation/control?repo=local/repo", timeout=5) as response: control = json.loads(response.read().decode("utf-8"))["control"] - self.assertIn(control["action"], {"continue", "review_only", "pause_auto_fix", "escalate"}) + self.assertIn(control["action"], {"continue", "review_only", "pause_auto_fix", "escalate", "defer"}) self.assertIn("execution", control) finally: server.shutdown() diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 013c11e2..8d1d78a1 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -134,7 +134,7 @@ def test_failure_streak_matching_is_case_insensitive(self) -> None: }, ] - self.assertEqual(consecutive_failure_count(runs, repo="quantstrategylab/aiauditbridge", task_name="monthly"), 1) + self.assertEqual(consecutive_failure_count(runs, repo="quantstrategylab/aiauditbridge"), 1) def test_external_workflow_failures_do_not_force_repo_failure_streak(self) -> None: runs = [ @@ -188,7 +188,7 @@ def test_consecutive_failures_force_human_review(self) -> None: policy={"default": {"max_consecutive_failures": 2}}, ) - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge", task_name="monthly"), 2) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 2) self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) @@ -206,7 +206,7 @@ def test_running_state_does_not_clear_failure_streak(self) -> None: }, ] - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge", task_name="monthly"), 1) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) def test_invalid_failure_threshold_falls_back_safely(self) -> None: result = decide_automation_execution( From 2afcb30a7eb5865af31a7a8770d3ffe985cea262 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:08:11 +0800 Subject: [PATCH 12/72] fix: keep automation control legacy actions compatible Co-Authored-By: Codex --- service/ai_gateway_service.py | 13 +++++-------- service/automation_run_ledger.py | 2 -- tests/test_ai_gateway_automation_control.py | 11 +++++------ tests/test_ai_gateway_service_get_routes.py | 2 +- 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index ddc7846d..0d881859 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -67,14 +67,13 @@ ) from service.automation_run_ledger import ( CONTROL_CONTINUE, - CONTROL_DEFER, CONTROL_ESCALATE, CONTROL_PAUSE_AUTO_FIX, CONTROL_REVIEW_ONLY, get_automation_run_ledger, suggest_control_action, ) -from service.automation_decision import EXECUTION_DEFER, EXECUTION_HUMAN_REVIEW, decide_automation_execution, load_execution_policy +from service.automation_decision import EXECUTION_HUMAN_REVIEW, decide_automation_execution, load_execution_policy from service.strategy_automation_registry import ( apply_strategy_registry_guard, summarize_strategy_registry_context, @@ -514,7 +513,7 @@ def _automation_control_snapshot( repo: str, *, task_name: str = "", - requested_mode: str = MODE_REVIEW_ONLY, + requested_mode: str = MODE_REVIEW_AND_FIX, pending_run: dict[str, Any] | None = None, ) -> dict[str, Any]: try: @@ -560,8 +559,6 @@ def _automation_control_snapshot( strict_action = original_action if execution.get("action") == EXECUTION_HUMAN_REVIEW: strict_action = CONTROL_ESCALATE - elif execution.get("action") == EXECUTION_DEFER: - strict_action = CONTROL_DEFER elif ( execution.get("requested_mode") == MODE_REVIEW_AND_FIX and execution.get("effective_mode") == MODE_REVIEW_ONLY @@ -571,7 +568,7 @@ def _automation_control_snapshot( if strict_action != original_action: control["action"] = strict_action control["auto_fix_allowed"] = False - control["requires_human_review"] = execution.get("action") != EXECUTION_DEFER + control["requires_human_review"] = True reasons = control.get("reasons") if isinstance(control.get("reasons"), list) else [] reasons.append("capped by execution decision") control["reasons"] = reasons @@ -1499,7 +1496,7 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - mode = str(params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY) + mode = str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX) _json_response(self, HTTPStatus.OK, {"status": "ok", "control": _automation_control_snapshot(repo, requested_mode=mode)}) def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, Any]) -> None: @@ -1604,7 +1601,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st control = _automation_control_snapshot( repo, task_name=task_name, - requested_mode=str(payload.get("mode") or MODE_REVIEW_ONLY), + requested_mode=str(payload.get("mode") or MODE_REVIEW_AND_FIX), pending_run={ "run_id": run_id, "task_name": task_name, diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index b778208f..4d1827b6 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -22,7 +22,6 @@ CONTROL_REVIEW_ONLY = "review_only" CONTROL_PAUSE_AUTO_FIX = "pause_auto_fix" CONTROL_ESCALATE = "escalate" -CONTROL_DEFER = "defer" CONTROL_ACTIONS = frozenset( { @@ -30,7 +29,6 @@ CONTROL_REVIEW_ONLY, CONTROL_PAUSE_AUTO_FIX, CONTROL_ESCALATE, - CONTROL_DEFER, } ) diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 1c05bba6..6dbbb813 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -13,7 +13,7 @@ class TestAutomationControlSnapshot(unittest.TestCase): - def test_control_snapshot_defaults_to_review_only_for_healthy_repo(self) -> None: + def test_control_snapshot_defaults_to_review_and_fix_for_healthy_repo(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -28,8 +28,8 @@ def test_control_snapshot_defaults_to_review_only_for_healthy_repo(self) -> None control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") self.assertEqual(control["action"], "continue") - self.assertEqual(control["execution"]["effective_mode"], "review_only") - self.assertFalse(control["execution"]["auto_fix_allowed"]) + self.assertEqual(control["execution"]["effective_mode"], "review_and_fix") + self.assertTrue(control["execution"]["auto_fix_allowed"]) def test_control_snapshot_preserves_continue_for_explicit_review_and_fix_mode(self) -> None: health = type("Health", (), {"status": "healthy"})() @@ -210,7 +210,7 @@ def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: self.assertEqual(control["action"], "continue") self.assertEqual(control["execution"]["consecutive_failures"], 1) - def test_control_snapshot_maps_defer_to_legacy_defer(self) -> None: + def test_control_snapshot_keeps_legacy_pause_for_defer(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "low"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -227,8 +227,7 @@ def test_control_snapshot_maps_defer_to_legacy_defer(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") - self.assertEqual(control["action"], "defer") - self.assertFalse(control["requires_human_review"]) + self.assertEqual(control["action"], "pause_auto_fix") self.assertEqual(control["execution"]["action"], "defer") def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index bc0ef1d4..c1b6ea36 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -580,7 +580,7 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: with urllib.request.urlopen(f"{base_url}/v1/ai/automation/control?repo=local/repo", timeout=5) as response: control = json.loads(response.read().decode("utf-8"))["control"] - self.assertIn(control["action"], {"continue", "review_only", "pause_auto_fix", "escalate", "defer"}) + self.assertIn(control["action"], {"continue", "review_only", "pause_auto_fix", "escalate"}) self.assertIn("execution", control) finally: server.shutdown() From 9b84ef2cdcd4edecf1d83bc3187af6ad0bd24ec3 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:14:38 +0800 Subject: [PATCH 13/72] fix: align defer with legacy pause control Co-Authored-By: Codex --- service/ai_gateway_service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 0d881859..ab2d7b8c 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -73,7 +73,7 @@ get_automation_run_ledger, suggest_control_action, ) -from service.automation_decision import EXECUTION_HUMAN_REVIEW, decide_automation_execution, load_execution_policy +from service.automation_decision import EXECUTION_DEFER, EXECUTION_HUMAN_REVIEW, decide_automation_execution, load_execution_policy from service.strategy_automation_registry import ( apply_strategy_registry_guard, summarize_strategy_registry_context, @@ -552,6 +552,7 @@ def _automation_control_snapshot( execution["effective_mode"] = MODE_REVIEW_ONLY execution["human_review_required"] = True execution["auto_fix_allowed"] = False + execution["defer"] = False reasons = execution.get("reasons") if isinstance(execution.get("reasons"), list) else [] reasons.append("automation ledger unavailable; forcing human review") execution["reasons"] = reasons @@ -559,6 +560,8 @@ def _automation_control_snapshot( strict_action = original_action if execution.get("action") == EXECUTION_HUMAN_REVIEW: strict_action = CONTROL_ESCALATE + elif execution.get("action") == EXECUTION_DEFER: + strict_action = CONTROL_PAUSE_AUTO_FIX elif ( execution.get("requested_mode") == MODE_REVIEW_AND_FIX and execution.get("effective_mode") == MODE_REVIEW_ONLY From a349cf63d3a6c3e60f5616ed9f49b7a86dbf1cfc Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:19:47 +0800 Subject: [PATCH 14/72] fix: validate control mode and failure streak gaps Co-Authored-By: Codex --- service/ai_gateway_service.py | 3 +++ service/automation_decision.py | 28 +++++++++++++-------- tests/test_ai_gateway_service_get_routes.py | 8 ++++++ tests/test_automation_decision.py | 17 ++++++++++++- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index ab2d7b8c..d57882ad 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -1500,6 +1500,9 @@ def _handle_automation_control(self) -> None: repo = claims_repo repo = repo or "unknown" mode = str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX) + if mode not in {MODE_REVIEW_ONLY, MODE_REVIEW_AND_FIX}: + _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) + return _json_response(self, HTTPStatus.OK, {"status": "ok", "control": _automation_control_snapshot(repo, requested_mode=mode)}) def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, Any]) -> None: diff --git a/service/automation_decision.py b/service/automation_decision.py index c199b981..93ffdae4 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -9,7 +9,6 @@ from service.automation_run_ledger import CONTROL_ESCALATE, CONTROL_PAUSE_AUTO_FIX, CONTROL_REVIEW_ONLY from service.quota import recommend_model -from service.task_state import TERMINAL_STATES EXECUTION_RUN = "run" EXECUTION_REVIEW_ONLY = "review_only" @@ -100,6 +99,8 @@ def _fail_closed_policy(reason: str) -> dict[str, Any]: "default": { "max_autonomy": AUTONOMY_MANUAL, "max_consecutive_failures": 1, + "low_cost_model": DEFAULT_LOW_COST_MODEL, + "low_cost_provider": DEFAULT_LOW_COST_PROVIDER, }, } @@ -157,11 +158,15 @@ def load_execution_policy(path: Path | None = None) -> dict[str, Any]: if not configured: return _fail_closed_policy("execution policy path is not configured") path = Path(configured).expanduser() - if not path.exists(): + try: + raw = path.read_text(encoding="utf-8") + except FileNotFoundError: return _fail_closed_policy("execution policy file is unavailable") + except OSError: + return _fail_closed_policy("execution policy file is unreadable") try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): + payload = json.loads(raw) + except json.JSONDecodeError: return _fail_closed_policy("execution policy file is unreadable") if not isinstance(payload, dict): return _fail_closed_policy("execution policy file is invalid") @@ -205,8 +210,7 @@ def consecutive_failure_count( if state == "failed": count += 1 continue - if state in TERMINAL_STATES: - break + break return count @@ -281,9 +285,9 @@ def decide_automation_execution( reasons.append("health unhealthy; forcing human review") if quota in {"low", "constrained"}: - effective_model = low_cost_model or recommend_model(0.0) - effective_provider = low_cost_provider or "auto" - if quota_low_behavior == "defer": + if action in {EXECUTION_HUMAN_REVIEW, EXECUTION_DEFER}: + reasons.append(f"quota status is {quota}; execution already blocked") + elif quota_low_behavior == "defer": if action != EXECUTION_HUMAN_REVIEW: action = EXECUTION_DEFER defer = True @@ -291,14 +295,18 @@ def decide_automation_execution( human_review_required = True reasons.append(f"quota status is {quota}; deferring automation") else: + effective_model = low_cost_model or recommend_model(0.0) + effective_provider = low_cost_provider or "auto" reasons.append(f"quota status is {quota}; recommending low-cost model") elif quota in {"exhausted", "blocked"}: if action != EXECUTION_HUMAN_REVIEW: action = EXECUTION_DEFER defer = True + reasons.append(f"quota status is {quota}; deferring automation") + else: + reasons.append(f"quota status is {quota}; execution already blocked") effective_mode = MODE_REVIEW_ONLY human_review_required = True - reasons.append(f"quota status is {quota}; deferring automation") return { "action": action, diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index c1b6ea36..6e365609 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -378,6 +378,14 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) + invalid_mode_request = urllib.request.Request( + f"{base_url}/v1/ai/automation/control?repo=QuantStrategyLab/TargetRepo&mode=bad", + headers={"Authorization": f"Bearer {token}"}, + ) + with self.assertRaises(urllib.error.HTTPError) as ctx: + urllib.request.urlopen(invalid_mode_request, timeout=5) + self.assertEqual(ctx.exception.code, 400) + missing_repo_request = urllib.request.Request( f"{base_url}/v1/ai/automation/control", headers={"Authorization": f"Bearer {token}"}, diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 8d1d78a1..0c557482 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -110,6 +110,21 @@ def test_low_quota_overrides_requested_expensive_model(self) -> None: self.assertEqual(result["effective_provider"], "openai") self.assertEqual(result["effective_model"], "gpt-5.4-mini") + def test_low_quota_does_not_override_model_when_human_review_required(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode=MODE_REVIEW_AND_FIX, + requested_model="gpt-5.4-pro", + control_action=CONTROL_ESCALATE, + service_health="healthy", + quota_status="low", + org_health_status="ok", + policy={"default": {"low_cost_model": "gpt-5.4-mini", "low_cost_provider": "openai"}}, + ) + + self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) + self.assertEqual(result["effective_model"], "gpt-5.4-pro") + def test_repo_policy_can_force_review_only(self) -> None: result = decide_automation_execution( repo="quantstrategylab/cryptolivepoolpipelines", @@ -206,7 +221,7 @@ def test_running_state_does_not_clear_failure_streak(self) -> None: }, ] - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 0) def test_invalid_failure_threshold_falls_back_safely(self) -> None: result = decide_automation_execution( From aa3edc78123fd46c9f3efa15019182a73d308292 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:27:24 +0800 Subject: [PATCH 15/72] fix: distinguish review-only from human review Co-Authored-By: Codex --- service/automation_decision.py | 10 +++++----- tests/test_automation_decision.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/service/automation_decision.py b/service/automation_decision.py index 93ffdae4..ae10c0ec 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -210,6 +210,8 @@ def consecutive_failure_count( if state == "failed": count += 1 continue + if state in {"queued", "running", "pending", "in_progress"}: + continue break return count @@ -252,7 +254,6 @@ def decide_automation_execution( if AUTONOMY_RANK[max_autonomy] <= AUTONOMY_RANK[AUTONOMY_REVIEW_ONLY]: effective_mode = MODE_REVIEW_ONLY - human_review_required = True reasons.append(f"repo max autonomy is {max_autonomy}") if autonomy_config_error: reasons.append(autonomy_config_error) @@ -260,6 +261,7 @@ def decide_automation_execution( reasons.append(policy_load_error) if max_autonomy == AUTONOMY_MANUAL: action = EXECUTION_HUMAN_REVIEW + human_review_required = True if failures >= max_failures: action = EXECUTION_HUMAN_REVIEW @@ -269,14 +271,13 @@ def decide_automation_execution( if control_action in {CONTROL_REVIEW_ONLY, CONTROL_PAUSE_AUTO_FIX, CONTROL_ESCALATE}: effective_mode = MODE_REVIEW_ONLY - human_review_required = True reasons.append(f"runtime control action is {control_action}") if control_action == CONTROL_ESCALATE: action = EXECUTION_HUMAN_REVIEW + human_review_required = True if service == "degraded" or org_health == "degraded": effective_mode = MODE_REVIEW_ONLY - human_review_required = True reasons.append("health degraded; forcing review_only") if service == "unhealthy" or org_health == "unhealthy": action = EXECUTION_HUMAN_REVIEW @@ -292,7 +293,6 @@ def decide_automation_execution( action = EXECUTION_DEFER defer = True effective_mode = MODE_REVIEW_ONLY - human_review_required = True reasons.append(f"quota status is {quota}; deferring automation") else: effective_model = low_cost_model or recommend_model(0.0) @@ -306,7 +306,7 @@ def decide_automation_execution( else: reasons.append(f"quota status is {quota}; execution already blocked") effective_mode = MODE_REVIEW_ONLY - human_review_required = True + human_review_required = action == EXECUTION_HUMAN_REVIEW return { "action": action, diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 0c557482..5d536c5d 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -49,7 +49,7 @@ def test_degraded_health_forces_review_only(self) -> None: self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) self.assertFalse(result["auto_fix_allowed"]) - self.assertTrue(result["human_review_required"]) + self.assertFalse(result["human_review_required"]) def test_exhausted_quota_defers_execution_without_stronger_guard(self) -> None: result = decide_automation_execution( @@ -137,7 +137,7 @@ def test_repo_policy_can_force_review_only(self) -> None: ) self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) - self.assertTrue(result["human_review_required"]) + self.assertFalse(result["human_review_required"]) self.assertFalse(result["auto_fix_allowed"]) def test_failure_streak_matching_is_case_insensitive(self) -> None: @@ -221,7 +221,7 @@ def test_running_state_does_not_clear_failure_streak(self) -> None: }, ] - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 0) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) def test_invalid_failure_threshold_falls_back_safely(self) -> None: result = decide_automation_execution( From 0d82924ca4064031cf7ca3b0b06316b8a12b5a03 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:35:34 +0800 Subject: [PATCH 16/72] fix: harden execution policy deployment Co-Authored-By: Codex --- scripts/deploy_codex_audit_service.sh | 35 ++++++++++++++++----- service/ai_gateway_service.py | 4 +-- tests/test_ai_gateway_automation_control.py | 2 ++ tests/test_run_monthly_codex_audit.py | 1 + 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/scripts/deploy_codex_audit_service.sh b/scripts/deploy_codex_audit_service.sh index 47bc691c..c90d7495 100644 --- a/scripts/deploy_codex_audit_service.sh +++ b/scripts/deploy_codex_audit_service.sh @@ -217,13 +217,23 @@ write_admin_env_file_if_needed() { write_default_execution_policy_if_missing() { local owner="$1" local policy_path="${JOB_DIR}/execution_policy.json" + if [ -L "$policy_path" ]; then + echo "refusing to write execution policy through symlink: $policy_path" >&2 + exit 1 + fi if [ -e "$policy_path" ]; then return fi - local tmp - tmp="$(mktemp)" - cat >"$tmp" <<'EOF_POLICY' -{ + sudo python3 - "$policy_path" "$owner" <<'PY' +import os +import pwd +import sys + +path, owner = sys.argv[1], sys.argv[2] +flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL +if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW +content = """{ "default": { "max_autonomy": "auto_pr", "max_consecutive_failures": 3, @@ -232,9 +242,20 @@ write_default_execution_policy_if_missing() { }, "repositories": {} } -EOF_POLICY - sudo install -m 0600 -o "$owner" -g "$owner" "$tmp" "$policy_path" - rm -f "$tmp" +""" +try: + fd = os.open(path, flags, 0o600) +except FileExistsError: + if os.path.islink(path): + print(f"refusing to write execution policy through symlink: {path}", file=sys.stderr) + raise SystemExit(1) + raise SystemExit(0) +with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(content) +user = pwd.getpwnam(owner) +os.chown(path, user.pw_uid, user.pw_gid) +os.chmod(path, 0o600) +PY } write_audit_service_unit() { diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index d57882ad..8821b749 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -570,11 +570,11 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY if strict_action != original_action: control["action"] = strict_action - control["auto_fix_allowed"] = False - control["requires_human_review"] = True reasons = control.get("reasons") if isinstance(control.get("reasons"), list) else [] reasons.append("capped by execution decision") control["reasons"] = reasons + control["auto_fix_allowed"] = bool(execution.get("auto_fix_allowed")) + control["requires_human_review"] = bool(execution.get("human_review_required")) control["execution"] = execution return control diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 6dbbb813..bff5f9d7 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -85,6 +85,7 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") self.assertEqual(control["action"], "review_only") + self.assertFalse(control["requires_human_review"]) self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) @@ -228,6 +229,7 @@ def test_control_snapshot_keeps_legacy_pause_for_defer(self) -> None: control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") self.assertEqual(control["action"], "pause_auto_fix") + self.assertFalse(control["requires_human_review"]) self.assertEqual(control["execution"]["action"], "defer") def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: diff --git a/tests/test_run_monthly_codex_audit.py b/tests/test_run_monthly_codex_audit.py index afd1bf93..c1aafff6 100644 --- a/tests/test_run_monthly_codex_audit.py +++ b/tests/test_run_monthly_codex_audit.py @@ -2344,6 +2344,7 @@ def test_vps_deploy_adds_nginx_audit_route_without_router_service(self) -> None: self.assertIn("CODEX_AUDIT_SERVICE_JOB_DIR", deploy_script) self.assertIn("CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH", deploy_script) self.assertIn("write_default_execution_policy_if_missing", deploy_script) + self.assertIn("O_NOFOLLOW", deploy_script) self.assertIn('"max_consecutive_failures": 3', deploy_script) self.assertIn("codex_pr_review.yml@refs/pull/*/merge", deploy_script) self.assertIn("refs/pull/*/merge", deploy_script) From f3f65503efe3d567df43c5cd7de1566831c440f4 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:42:10 +0800 Subject: [PATCH 17/72] fix: accept legacy control modes Co-Authored-By: Codex --- service/ai_gateway_service.py | 13 +++++++++++-- tests/test_ai_gateway_service_get_routes.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 8821b749..56990d27 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -579,6 +579,15 @@ def _automation_control_snapshot( return control +def _normalize_control_mode_param(value: str) -> str: + mode = str(value or "").strip().lower() + if mode in {MODE_REVIEW_ONLY, "manual"}: + return MODE_REVIEW_ONLY + if mode in {MODE_REVIEW_AND_FIX, "auto_pr", "auto_merge"}: + return MODE_REVIEW_AND_FIX + return "" + + def _highest_changed_path_risk(changed_paths: list[str], policy: dict[str, Any]) -> str: if not changed_paths: return RISK_LOW @@ -1499,8 +1508,8 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - mode = str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX) - if mode not in {MODE_REVIEW_ONLY, MODE_REVIEW_AND_FIX}: + mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX)) + if not mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return _json_response(self, HTTPStatus.OK, {"status": "ok", "control": _automation_control_snapshot(repo, requested_mode=mode)}) diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 6e365609..d317b06d 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -378,6 +378,16 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) + for legacy_mode, expected_mode in {"manual": "review_only", "auto_pr": "review_and_fix", "auto_merge": "review_and_fix"}.items(): + legacy_mode_request = urllib.request.Request( + f"{base_url}/v1/ai/automation/control?repo=QuantStrategyLab/TargetRepo&mode={legacy_mode}", + headers={"Authorization": f"Bearer {token}"}, + ) + with urllib.request.urlopen(legacy_mode_request, timeout=5) as response: + self.assertEqual(response.status, 200) + legacy_control = json.loads(response.read().decode("utf-8"))["control"] + self.assertEqual(legacy_control["execution"]["requested_mode"], expected_mode) + invalid_mode_request = urllib.request.Request( f"{base_url}/v1/ai/automation/control?repo=QuantStrategyLab/TargetRepo&mode=bad", headers={"Authorization": f"Bearer {token}"}, From f79bf09d5272f3d64a3854a363040a8173a4f88d Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:47:44 +0800 Subject: [PATCH 18/72] fix: normalize legacy execution modes Co-Authored-By: Codex --- service/automation_decision.py | 7 ++++++- tests/test_automation_decision.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/service/automation_decision.py b/service/automation_decision.py index ae10c0ec..0711c9b3 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -64,7 +64,10 @@ def _normalize_quota_status(value: Any, default: str = "unknown") -> str: def _normalize_mode(value: str) -> str: - return MODE_REVIEW_AND_FIX if str(value or "").strip() == MODE_REVIEW_AND_FIX else MODE_REVIEW_ONLY + mode = str(value or "").strip().lower() + if mode in {MODE_REVIEW_AND_FIX, AUTONOMY_AUTO_PR, AUTONOMY_AUTO_MERGE}: + return MODE_REVIEW_AND_FIX + return MODE_REVIEW_ONLY def _normalize_repo_id(value: Any) -> str: @@ -162,6 +165,8 @@ def load_execution_policy(path: Path | None = None) -> dict[str, Any]: raw = path.read_text(encoding="utf-8") except FileNotFoundError: return _fail_closed_policy("execution policy file is unavailable") + except UnicodeDecodeError: + return _fail_closed_policy("execution policy file is unreadable") except OSError: return _fail_closed_policy("execution policy file is unreadable") try: diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 5d536c5d..3dd1c096 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -37,6 +37,22 @@ def test_healthy_control_allows_review_and_fix(self) -> None: self.assertEqual(result["effective_mode"], MODE_REVIEW_AND_FIX) self.assertTrue(result["auto_fix_allowed"]) + def test_legacy_autonomy_modes_are_normalized(self) -> None: + for requested_mode, expected_mode in { + "manual": MODE_REVIEW_ONLY, + "auto_pr": MODE_REVIEW_AND_FIX, + "auto_merge": MODE_REVIEW_AND_FIX, + }.items(): + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode=requested_mode, + control_action=CONTROL_CONTINUE, + service_health="healthy", + quota_status="ok", + org_health_status="ok", + ) + self.assertEqual(result["requested_mode"], expected_mode) + def test_degraded_health_forces_review_only(self) -> None: result = decide_automation_execution( repo="QuantStrategyLab/AIAuditBridge", @@ -247,6 +263,9 @@ def test_load_execution_policy_fails_closed_for_malformed_files(self) -> None: path.write_text(json.dumps({"default": []}), encoding="utf-8") self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "manual") + path.write_bytes(b"\xff\xfe\x00") + self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "manual") + path.write_text(json.dumps({"repositories": {"QuantStrategyLab/AIAuditBridge": "bad"}}), encoding="utf-8") self.assertEqual(load_execution_policy(path)["default"]["max_autonomy"], "manual") From be0023c3bde14fc892308370d48a1819f1205ff7 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 04:54:22 +0800 Subject: [PATCH 19/72] fix: keep execution policy admin owned Co-Authored-By: Codex --- docs/async_service_deployment.md | 7 ++-- scripts/deploy_codex_audit_service.sh | 21 ++++++------ service/automation_decision.py | 37 ++++++++++++++++++++- tests/test_ai_gateway_automation_control.py | 9 ++++- tests/test_automation_decision.py | 20 +++++++++++ tests/test_run_monthly_codex_audit.py | 1 + 6 files changed, 79 insertions(+), 16 deletions(-) diff --git a/docs/async_service_deployment.md b/docs/async_service_deployment.md index 06eb6932..dbd4d201 100644 --- a/docs/async_service_deployment.md +++ b/docs/async_service_deployment.md @@ -72,9 +72,10 @@ bash scripts/deploy_codex_audit_service.sh deploy The job directory should be owned by the service user and mode `0700`. The deploy script points `CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH` to -`${CODEX_AUDIT_SERVICE_JOB_DIR}/execution_policy.json` and creates a conservative -default file if it is missing. This service-owned file can cap repo autonomy -without trusting the reviewed checkout: +`/var/lib/codex-audit-bridge/policy/execution_policy.json` by default and creates +a conservative default file if it is missing. This admin-owned, service-readable +policy file sits outside the service-user-writable job workspace and can cap repo +autonomy without trusting the reviewed checkout: ```json { diff --git a/scripts/deploy_codex_audit_service.sh b/scripts/deploy_codex_audit_service.sh index c90d7495..7f10ac52 100644 --- a/scripts/deploy_codex_audit_service.sh +++ b/scripts/deploy_codex_audit_service.sh @@ -14,6 +14,7 @@ ALLOWED_REPOSITORY_VISIBILITIES="${CODEX_AUDIT_SERVICE_ALLOWED_REPOSITORY_VISIBI ALLOWED_SOURCE_REPOSITORIES="${CODEX_AUDIT_SERVICE_ALLOWED_SOURCE_REPOSITORIES:-QuantStrategyLab/AIAuditBridge,QuantStrategyLab/CryptoLivePoolPipelines,QuantStrategyLab/HkEquitySnapshotPipelines,QuantStrategyLab/UsEquitySnapshotPipelines,QuantStrategyLab/ResearchSignalContextPipelines}" JOB_DIR="${CODEX_AUDIT_SERVICE_JOB_DIR:-/var/lib/codex-audit-bridge/jobs}" ADMIN_ENV_FILE="${CODEX_AUDIT_SERVICE_ADMIN_ENV_FILE:-/etc/codex-audit-bridge/admin.env}" +EXECUTION_POLICY_FILE="${CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH:-/var/lib/codex-audit-bridge/policy/execution_policy.json}" AUDIT_MODEL="${CODEX_AUDIT_SERVICE_MODEL:-}" AUDIT_REASONING_EFFORT="${CODEX_AUDIT_SERVICE_REASONING_EFFORT:-}" CODEX_ACCOUNT_USAGE="${CODEX_AUDIT_SERVICE_CODEX_ACCOUNT_USAGE:-1}" @@ -215,8 +216,7 @@ write_admin_env_file_if_needed() { } write_default_execution_policy_if_missing() { - local owner="$1" - local policy_path="${JOB_DIR}/execution_policy.json" + local policy_path="${EXECUTION_POLICY_FILE}" if [ -L "$policy_path" ]; then echo "refusing to write execution policy through symlink: $policy_path" >&2 exit 1 @@ -224,12 +224,12 @@ write_default_execution_policy_if_missing() { if [ -e "$policy_path" ]; then return fi - sudo python3 - "$policy_path" "$owner" <<'PY' + sudo install -d -m 0755 -o root -g root "$(dirname "$policy_path")" + sudo python3 - "$policy_path" <<'PY' import os -import pwd import sys -path, owner = sys.argv[1], sys.argv[2] +path = sys.argv[1] flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW @@ -252,9 +252,8 @@ except FileExistsError: raise SystemExit(0) with os.fdopen(fd, "w", encoding="utf-8") as handle: handle.write(content) -user = pwd.getpwnam(owner) -os.chown(path, user.pw_uid, user.pw_gid) -os.chmod(path, 0o600) +os.chown(path, 0, 0) +os.chmod(path, 0o644) PY } @@ -296,7 +295,7 @@ Environment=CODEX_AUDIT_SERVICE_ALLOWED_REPOSITORY_VISIBILITIES=${ALLOWED_REPOSI Environment=CODEX_AUDIT_SERVICE_ALLOWED_SOURCE_REPOSITORIES=${ALLOWED_SOURCE_REPOSITORIES} Environment=CODEX_AUDIT_SERVICE_JOB_DIR=${JOB_DIR} Environment=CODEX_AUDIT_SERVICE_QUOTA_STORE=${JOB_DIR}/quota.json -Environment=CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH=${JOB_DIR}/execution_policy.json +Environment=CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH=${EXECUTION_POLICY_FILE} Environment=CODEX_AUDIT_SERVICE_CODEX_ACCOUNT_USAGE=${CODEX_ACCOUNT_USAGE} Environment=CODEX_AUDIT_SERVICE_OPENAI_USAGE_WINDOW_DAYS=${OPENAI_USAGE_WINDOW_DAYS} Environment=CODEX_AUDIT_SERVICE_ANTHROPIC_USAGE_WINDOW_DAYS=${ANTHROPIC_USAGE_WINDOW_DAYS} @@ -326,7 +325,7 @@ Environment="CODEX_AUDIT_SERVICE_ALLOWED_WORKFLOW_REFS=${ALLOWED_WORKFLOW_REFS}" Environment="CODEX_AUDIT_SERVICE_ALLOWED_REFS=${ALLOWED_REFS}" Environment="CODEX_AUDIT_SERVICE_ALLOWED_REPOSITORY_VISIBILITIES=${ALLOWED_REPOSITORY_VISIBILITIES}" Environment="CODEX_AUDIT_SERVICE_ALLOWED_SOURCE_REPOSITORIES=${ALLOWED_SOURCE_REPOSITORIES}" -Environment="CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH=${JOB_DIR}/execution_policy.json" +Environment="CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH=${EXECUTION_POLICY_FILE}" EOF_DROPIN } @@ -554,7 +553,7 @@ deploy() { install_file "scripts/codex_audit_service.py" "${DEPLOY_DIR}/scripts/codex_audit_service.py" "0755" install_service_package sudo install -d -m 0700 -o "$runner_user" -g "$runner_user" "$JOB_DIR" - write_default_execution_policy_if_missing "$runner_user" + write_default_execution_policy_if_missing write_admin_env_file_if_needed write_audit_service_unit write_managed_audit_service_dropin diff --git a/service/automation_decision.py b/service/automation_decision.py index 0711c9b3..3fed4da0 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -5,6 +5,7 @@ import json import os from pathlib import Path +import stat from typing import Any from service.automation_run_ledger import CONTROL_ESCALATE, CONTROL_PAUSE_AUTO_FIX, CONTROL_REVIEW_ONLY @@ -29,6 +30,7 @@ DEFAULT_LOW_COST_MODEL = "gpt-5.4-mini" DEFAULT_LOW_COST_PROVIDER = "openai" EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH" +EXECUTION_POLICY_OWNER_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_OWNER" POLICY_LOAD_ERROR_KEY = "_load_error" TRUSTED_FAILURE_ORIGINS = frozenset({"service_job"}) POLICY_ALLOWED_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider", "quota_low_behavior"}) @@ -108,6 +110,34 @@ def _fail_closed_policy(reason: str) -> dict[str, Any]: } +def _expected_policy_owner() -> tuple[int, int]: + raw = os.environ.get(EXECUTION_POLICY_OWNER_ENV, "0:0").strip() or "0:0" + try: + uid, gid = raw.split(":", 1) + return int(uid), int(gid) + except (TypeError, ValueError): + return 0, 0 + + +def _policy_trust_error(path: Path) -> str: + try: + info = path.lstat() + except FileNotFoundError: + return "execution policy file is unavailable" + except OSError: + return "execution policy file is unreadable" + if stat.S_ISLNK(info.st_mode): + return "execution policy file is a symlink" + if not stat.S_ISREG(info.st_mode): + return "execution policy file is not a regular file" + expected_uid, expected_gid = _expected_policy_owner() + if (info.st_uid, info.st_gid) != (expected_uid, expected_gid): + return "execution policy file owner is invalid" + if info.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + return "execution policy file permissions are too broad" + return "" + + def _validate_execution_policy(payload: dict[str, Any]) -> str: default_policy = payload.get("default") if not isinstance(default_policy, dict): @@ -155,12 +185,17 @@ def _validate_policy_section(section: dict[str, Any], *, section_name: str, requ def load_execution_policy(path: Path | None = None) -> dict[str, Any]: - """Load service-owned execution policy for repo autonomy thresholds.""" + """Load admin-owned execution policy for repo autonomy thresholds.""" + require_trusted_path = path is None if path is None: configured = os.environ.get(EXECUTION_POLICY_PATH_ENV, "").strip() if not configured: return _fail_closed_policy("execution policy path is not configured") path = Path(configured).expanduser() + if require_trusted_path: + trust_error = _policy_trust_error(path) + if trust_error: + return _fail_closed_policy(trust_error) try: raw = path.read_text(encoding="utf-8") except FileNotFoundError: diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index bff5f9d7..cd8f4c9e 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -76,7 +76,14 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: ledger = type("Ledger", (), {"snapshot": lambda self, limit=20: {"runs": []}})() with ( - patch.dict(os.environ, {"CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH": str(policy_path)}, clear=False), + patch.dict( + os.environ, + { + "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH": str(policy_path), + "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_OWNER": f"{os.getuid()}:{os.getgid()}", + }, + clear=False, + ), patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), patch("service.ai_gateway_service.get_health_monitor", return_value=health), patch("service.ai_gateway_service.get_quota_manager", return_value=quota), diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 3dd1c096..a57ec564 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -295,6 +295,26 @@ def test_load_execution_policy_fails_closed_when_path_unset(self) -> None: with patch.dict(os.environ, {}, clear=True): self.assertEqual(load_execution_policy()["default"]["max_autonomy"], "manual") + def test_load_execution_policy_fails_closed_for_untrusted_env_path(self) -> None: + with TemporaryDirectory() as tmp: + path = Path(tmp) / "policy.json" + path.write_text( + json.dumps( + { + "default": { + "max_autonomy": "auto_pr", + "max_consecutive_failures": 3, + "low_cost_model": "gpt-5.4-mini", + "low_cost_provider": "openai", + }, + "repositories": {}, + } + ), + encoding="utf-8", + ) + with patch.dict(os.environ, {"CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH": str(path)}, clear=False): + self.assertEqual(load_execution_policy()["default"]["max_autonomy"], "manual") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_run_monthly_codex_audit.py b/tests/test_run_monthly_codex_audit.py index c1aafff6..42824cb4 100644 --- a/tests/test_run_monthly_codex_audit.py +++ b/tests/test_run_monthly_codex_audit.py @@ -2343,6 +2343,7 @@ def test_vps_deploy_adds_nginx_audit_route_without_router_service(self) -> None: self.assertIn("location ^~ /v1/codex-audit/", deploy_script) self.assertIn("CODEX_AUDIT_SERVICE_JOB_DIR", deploy_script) self.assertIn("CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH", deploy_script) + self.assertIn("/var/lib/codex-audit-bridge/policy/execution_policy.json", deploy_script) self.assertIn("write_default_execution_policy_if_missing", deploy_script) self.assertIn("O_NOFOLLOW", deploy_script) self.assertIn('"max_consecutive_failures": 3', deploy_script) From dbb713a58bdb5ce3005b7b1d1416dd6247cb4e59 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:00:47 +0800 Subject: [PATCH 20/72] fix: cap requested autonomy level Co-Authored-By: Codex --- service/ai_gateway_service.py | 6 ++---- service/automation_decision.py | 25 +++++++++++++++++++++++-- tests/test_automation_decision.py | 28 +++++++++++++++++++++++----- 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 56990d27..44045b12 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -581,10 +581,8 @@ def _automation_control_snapshot( def _normalize_control_mode_param(value: str) -> str: mode = str(value or "").strip().lower() - if mode in {MODE_REVIEW_ONLY, "manual"}: - return MODE_REVIEW_ONLY - if mode in {MODE_REVIEW_AND_FIX, "auto_pr", "auto_merge"}: - return MODE_REVIEW_AND_FIX + if mode in {MODE_REVIEW_ONLY, MODE_REVIEW_AND_FIX, "manual", "auto_pr", "auto_merge"}: + return mode return "" diff --git a/service/automation_decision.py b/service/automation_decision.py index 3fed4da0..c8fb1865 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -72,6 +72,17 @@ def _normalize_mode(value: str) -> str: return MODE_REVIEW_ONLY +def _normalize_requested_autonomy(value: str) -> str: + mode = str(value or "").strip().lower() + if mode == AUTONOMY_AUTO_MERGE: + return AUTONOMY_AUTO_MERGE + if mode in {MODE_REVIEW_AND_FIX, AUTONOMY_AUTO_PR}: + return AUTONOMY_AUTO_PR + if mode == AUTONOMY_MANUAL: + return AUTONOMY_MANUAL + return AUTONOMY_REVIEW_ONLY + + def _normalize_repo_id(value: Any) -> str: return str(value or "").strip().lower() @@ -279,11 +290,18 @@ def decide_automation_execution( low_cost_provider = str(repo_policy.get("low_cost_provider") or DEFAULT_LOW_COST_PROVIDER).strip().lower() quota_low_behavior = str(repo_policy.get("quota_low_behavior") or "low_cost_model").strip().lower() + reasons: list[str] = [] + requested_autonomy = _normalize_requested_autonomy(requested_mode) + effective_autonomy = requested_autonomy + if AUTONOMY_RANK[effective_autonomy] > AUTONOMY_RANK[max_autonomy]: + effective_autonomy = max_autonomy + reasons.append(f"requested autonomy {requested_autonomy} exceeds repo max autonomy {max_autonomy}; capping") effective_mode = _normalize_mode(requested_mode) + if AUTONOMY_RANK[effective_autonomy] <= AUTONOMY_RANK[AUTONOMY_REVIEW_ONLY]: + effective_mode = MODE_REVIEW_ONLY effective_provider = str(requested_provider or "auto").strip().lower() or "auto" effective_model = str(requested_model or "").strip() action = EXECUTION_RUN - reasons: list[str] = [] human_review_required = False defer = False @@ -299,7 +317,7 @@ def decide_automation_execution( reasons.append(autonomy_config_error) if policy_load_error: reasons.append(policy_load_error) - if max_autonomy == AUTONOMY_MANUAL: + if effective_autonomy == AUTONOMY_MANUAL: action = EXECUTION_HUMAN_REVIEW human_review_required = True @@ -354,6 +372,8 @@ def decide_automation_execution( "task_name": task_name, "requested_mode": _normalize_mode(requested_mode), "effective_mode": effective_mode, + "requested_autonomy": requested_autonomy, + "effective_autonomy": effective_autonomy, "requested_provider": requested_provider, "effective_provider": effective_provider, "requested_model": requested_model, @@ -363,6 +383,7 @@ def decide_automation_execution( "max_consecutive_failures": max_failures, "human_review_required": human_review_required, "auto_fix_allowed": action == EXECUTION_RUN and effective_mode == MODE_REVIEW_AND_FIX and not human_review_required, + "auto_merge_allowed": action == EXECUTION_RUN and effective_autonomy == AUTONOMY_AUTO_MERGE and not human_review_required, "defer": defer, "reasons": reasons or ["execution allowed"], } diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index a57ec564..c45d2369 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -38,11 +38,12 @@ def test_healthy_control_allows_review_and_fix(self) -> None: self.assertTrue(result["auto_fix_allowed"]) def test_legacy_autonomy_modes_are_normalized(self) -> None: - for requested_mode, expected_mode in { - "manual": MODE_REVIEW_ONLY, - "auto_pr": MODE_REVIEW_AND_FIX, - "auto_merge": MODE_REVIEW_AND_FIX, - }.items(): + cases = { + "manual": (MODE_REVIEW_ONLY, "manual", "manual", False), + "auto_pr": (MODE_REVIEW_AND_FIX, "auto_pr", "auto_pr", False), + "auto_merge": (MODE_REVIEW_AND_FIX, "auto_merge", "auto_pr", False), + } + for requested_mode, (expected_mode, requested_autonomy, effective_autonomy, auto_merge_allowed) in cases.items(): result = decide_automation_execution( repo="QuantStrategyLab/AIAuditBridge", requested_mode=requested_mode, @@ -52,6 +53,23 @@ def test_legacy_autonomy_modes_are_normalized(self) -> None: org_health_status="ok", ) self.assertEqual(result["requested_mode"], expected_mode) + self.assertEqual(result["requested_autonomy"], requested_autonomy) + self.assertEqual(result["effective_autonomy"], effective_autonomy) + self.assertEqual(result["auto_merge_allowed"], auto_merge_allowed) + + def test_auto_merge_requires_matching_repo_autonomy(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode="auto_merge", + control_action=CONTROL_CONTINUE, + service_health="healthy", + quota_status="ok", + org_health_status="ok", + policy={"default": {"max_autonomy": "auto_merge"}}, + ) + + self.assertEqual(result["effective_autonomy"], "auto_merge") + self.assertTrue(result["auto_merge_allowed"]) def test_degraded_health_forces_review_only(self) -> None: result = decide_automation_execution( From 23ad52785dbda417ce23f043f2135bcdf02d9bdb Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:06:38 +0800 Subject: [PATCH 21/72] fix: keep control default review only Co-Authored-By: Codex --- service/ai_gateway_service.py | 2 +- service/automation_decision.py | 85 +++++++++++++++------ tests/test_ai_gateway_service_get_routes.py | 1 + 3 files changed, 63 insertions(+), 25 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 44045b12..c0b4a7a5 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -1506,7 +1506,7 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX)) + mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY)) if not mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return diff --git a/service/automation_decision.py b/service/automation_decision.py index c8fb1865..c23fdaed 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -130,25 +130,61 @@ def _expected_policy_owner() -> tuple[int, int]: return 0, 0 -def _policy_trust_error(path: Path) -> str: - try: - info = path.lstat() - except FileNotFoundError: - return "execution policy file is unavailable" - except OSError: - return "execution policy file is unreadable" - if stat.S_ISLNK(info.st_mode): - return "execution policy file is a symlink" - if not stat.S_ISREG(info.st_mode): - return "execution policy file is not a regular file" +def _policy_metadata_trust_error(info: os.stat_result, *, kind: str) -> str: expected_uid, expected_gid = _expected_policy_owner() if (info.st_uid, info.st_gid) != (expected_uid, expected_gid): - return "execution policy file owner is invalid" + return f"execution policy {kind} owner is invalid" if info.st_mode & (stat.S_IWGRP | stat.S_IWOTH): - return "execution policy file permissions are too broad" + return f"execution policy {kind} permissions are too broad" return "" +def _policy_parent_trust_error(path: Path) -> str: + try: + info = path.parent.lstat() + except FileNotFoundError: + return "execution policy parent directory is unavailable" + except OSError: + return "execution policy parent directory is unreadable" + if stat.S_ISLNK(info.st_mode): + return "execution policy parent directory is a symlink" + if not stat.S_ISDIR(info.st_mode): + return "execution policy parent path is not a directory" + return _policy_metadata_trust_error(info, kind="parent directory") + + +def _read_trusted_policy_file(path: Path) -> tuple[str, str]: + parent_error = _policy_parent_trust_error(path) + if parent_error: + return "", parent_error + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + fd = os.open(path, flags) + except FileNotFoundError: + return "", "execution policy file is unavailable" + except OSError: + return "", "execution policy file is unreadable" + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode): + return "", "execution policy file is not a regular file" + trust_error = _policy_metadata_trust_error(info, kind="file") + if trust_error: + return "", trust_error + with os.fdopen(fd, "r", encoding="utf-8") as handle: + fd = -1 + return handle.read(), "" + except UnicodeDecodeError: + return "", "execution policy file is unreadable" + except OSError: + return "", "execution policy file is unreadable" + finally: + if fd >= 0: + os.close(fd) + + def _validate_execution_policy(payload: dict[str, Any]) -> str: default_policy = payload.get("default") if not isinstance(default_policy, dict): @@ -204,17 +240,18 @@ def load_execution_policy(path: Path | None = None) -> dict[str, Any]: return _fail_closed_policy("execution policy path is not configured") path = Path(configured).expanduser() if require_trusted_path: - trust_error = _policy_trust_error(path) - if trust_error: - return _fail_closed_policy(trust_error) - try: - raw = path.read_text(encoding="utf-8") - except FileNotFoundError: - return _fail_closed_policy("execution policy file is unavailable") - except UnicodeDecodeError: - return _fail_closed_policy("execution policy file is unreadable") - except OSError: - return _fail_closed_policy("execution policy file is unreadable") + raw, read_error = _read_trusted_policy_file(path) + if read_error: + return _fail_closed_policy(read_error) + else: + try: + raw = path.read_text(encoding="utf-8") + except FileNotFoundError: + return _fail_closed_policy("execution policy file is unavailable") + except UnicodeDecodeError: + return _fail_closed_policy("execution policy file is unreadable") + except OSError: + return _fail_closed_policy("execution policy file is unreadable") try: payload = json.loads(raw) except json.JSONDecodeError: diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index d317b06d..8a20c63a 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -375,6 +375,7 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None control = json.loads(response.read().decode("utf-8"))["control"] self.assertIn("execution", control) self.assertEqual(control["execution"]["repo"], "QuantStrategyLab/TargetRepo") + self.assertEqual(control["execution"]["requested_mode"], "review_only") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) From e216952d83fc66d90ba401f53c33343abcf42709 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:13:33 +0800 Subject: [PATCH 22/72] fix: preserve manual as review only Co-Authored-By: Codex --- docs/async_service_deployment.md | 2 +- scripts/deploy_codex_audit_service.sh | 10 +++++-- service/ai_gateway_service.py | 2 +- service/automation_decision.py | 32 ++++++++++++--------- tests/test_ai_gateway_service_get_routes.py | 2 +- tests/test_automation_decision.py | 3 +- tests/test_run_monthly_codex_audit.py | 2 +- 7 files changed, 33 insertions(+), 20 deletions(-) diff --git a/docs/async_service_deployment.md b/docs/async_service_deployment.md index dbd4d201..1914dbe5 100644 --- a/docs/async_service_deployment.md +++ b/docs/async_service_deployment.md @@ -72,7 +72,7 @@ bash scripts/deploy_codex_audit_service.sh deploy The job directory should be owned by the service user and mode `0700`. The deploy script points `CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH` to -`/var/lib/codex-audit-bridge/policy/execution_policy.json` by default and creates +`/etc/codex-audit-bridge-policy/execution_policy.json` by default and creates a conservative default file if it is missing. This admin-owned, service-readable policy file sits outside the service-user-writable job workspace and can cap repo autonomy without trusting the reviewed checkout: diff --git a/scripts/deploy_codex_audit_service.sh b/scripts/deploy_codex_audit_service.sh index 7f10ac52..cbbe8aff 100644 --- a/scripts/deploy_codex_audit_service.sh +++ b/scripts/deploy_codex_audit_service.sh @@ -14,7 +14,7 @@ ALLOWED_REPOSITORY_VISIBILITIES="${CODEX_AUDIT_SERVICE_ALLOWED_REPOSITORY_VISIBI ALLOWED_SOURCE_REPOSITORIES="${CODEX_AUDIT_SERVICE_ALLOWED_SOURCE_REPOSITORIES:-QuantStrategyLab/AIAuditBridge,QuantStrategyLab/CryptoLivePoolPipelines,QuantStrategyLab/HkEquitySnapshotPipelines,QuantStrategyLab/UsEquitySnapshotPipelines,QuantStrategyLab/ResearchSignalContextPipelines}" JOB_DIR="${CODEX_AUDIT_SERVICE_JOB_DIR:-/var/lib/codex-audit-bridge/jobs}" ADMIN_ENV_FILE="${CODEX_AUDIT_SERVICE_ADMIN_ENV_FILE:-/etc/codex-audit-bridge/admin.env}" -EXECUTION_POLICY_FILE="${CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH:-/var/lib/codex-audit-bridge/policy/execution_policy.json}" +EXECUTION_POLICY_FILE="${CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH:-/etc/codex-audit-bridge-policy/execution_policy.json}" AUDIT_MODEL="${CODEX_AUDIT_SERVICE_MODEL:-}" AUDIT_REASONING_EFFORT="${CODEX_AUDIT_SERVICE_REASONING_EFFORT:-}" CODEX_ACCOUNT_USAGE="${CODEX_AUDIT_SERVICE_CODEX_ACCOUNT_USAGE:-1}" @@ -217,6 +217,12 @@ write_admin_env_file_if_needed() { write_default_execution_policy_if_missing() { local policy_path="${EXECUTION_POLICY_FILE}" + local policy_dir + policy_dir="$(dirname "$policy_path")" + if [ -L "$policy_dir" ]; then + echo "refusing to write execution policy under symlinked directory: $policy_dir" >&2 + exit 1 + fi if [ -L "$policy_path" ]; then echo "refusing to write execution policy through symlink: $policy_path" >&2 exit 1 @@ -224,7 +230,7 @@ write_default_execution_policy_if_missing() { if [ -e "$policy_path" ]; then return fi - sudo install -d -m 0755 -o root -g root "$(dirname "$policy_path")" + sudo install -d -m 0755 -o root -g root "$policy_dir" sudo python3 - "$policy_path" <<'PY' import os import sys diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index c0b4a7a5..44045b12 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -1506,7 +1506,7 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY)) + mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX)) if not mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return diff --git a/service/automation_decision.py b/service/automation_decision.py index c23fdaed..45fdafe7 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -78,8 +78,6 @@ def _normalize_requested_autonomy(value: str) -> str: return AUTONOMY_AUTO_MERGE if mode in {MODE_REVIEW_AND_FIX, AUTONOMY_AUTO_PR}: return AUTONOMY_AUTO_PR - if mode == AUTONOMY_MANUAL: - return AUTONOMY_MANUAL return AUTONOMY_REVIEW_ONLY @@ -140,17 +138,25 @@ def _policy_metadata_trust_error(info: os.stat_result, *, kind: str) -> str: def _policy_parent_trust_error(path: Path) -> str: - try: - info = path.parent.lstat() - except FileNotFoundError: - return "execution policy parent directory is unavailable" - except OSError: - return "execution policy parent directory is unreadable" - if stat.S_ISLNK(info.st_mode): - return "execution policy parent directory is a symlink" - if not stat.S_ISDIR(info.st_mode): - return "execution policy parent path is not a directory" - return _policy_metadata_trust_error(info, kind="parent directory") + expected_uid, expected_gid = _expected_policy_owner() + parents = [path.parent] + if expected_uid == 0 and expected_gid == 0: + parents.extend(path.parent.parents) + for parent in parents: + try: + info = parent.lstat() + except FileNotFoundError: + return "execution policy parent directory is unavailable" + except OSError: + return "execution policy parent directory is unreadable" + if stat.S_ISLNK(info.st_mode): + return "execution policy parent directory is a symlink" + if not stat.S_ISDIR(info.st_mode): + return "execution policy parent path is not a directory" + trust_error = _policy_metadata_trust_error(info, kind="parent directory") + if trust_error: + return trust_error + return "" def _read_trusted_policy_file(path: Path) -> tuple[str, str]: diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 8a20c63a..6507ac4a 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -375,7 +375,7 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None control = json.loads(response.read().decode("utf-8"))["control"] self.assertIn("execution", control) self.assertEqual(control["execution"]["repo"], "QuantStrategyLab/TargetRepo") - self.assertEqual(control["execution"]["requested_mode"], "review_only") + self.assertEqual(control["execution"]["requested_mode"], "review_and_fix") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index c45d2369..25bd927b 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -39,7 +39,7 @@ def test_healthy_control_allows_review_and_fix(self) -> None: def test_legacy_autonomy_modes_are_normalized(self) -> None: cases = { - "manual": (MODE_REVIEW_ONLY, "manual", "manual", False), + "manual": (MODE_REVIEW_ONLY, "review_only", "review_only", False), "auto_pr": (MODE_REVIEW_AND_FIX, "auto_pr", "auto_pr", False), "auto_merge": (MODE_REVIEW_AND_FIX, "auto_merge", "auto_pr", False), } @@ -56,6 +56,7 @@ def test_legacy_autonomy_modes_are_normalized(self) -> None: self.assertEqual(result["requested_autonomy"], requested_autonomy) self.assertEqual(result["effective_autonomy"], effective_autonomy) self.assertEqual(result["auto_merge_allowed"], auto_merge_allowed) + self.assertFalse(result["human_review_required"]) def test_auto_merge_requires_matching_repo_autonomy(self) -> None: result = decide_automation_execution( diff --git a/tests/test_run_monthly_codex_audit.py b/tests/test_run_monthly_codex_audit.py index 42824cb4..5f0112a2 100644 --- a/tests/test_run_monthly_codex_audit.py +++ b/tests/test_run_monthly_codex_audit.py @@ -2343,7 +2343,7 @@ def test_vps_deploy_adds_nginx_audit_route_without_router_service(self) -> None: self.assertIn("location ^~ /v1/codex-audit/", deploy_script) self.assertIn("CODEX_AUDIT_SERVICE_JOB_DIR", deploy_script) self.assertIn("CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH", deploy_script) - self.assertIn("/var/lib/codex-audit-bridge/policy/execution_policy.json", deploy_script) + self.assertIn("/etc/codex-audit-bridge-policy/execution_policy.json", deploy_script) self.assertIn("write_default_execution_policy_if_missing", deploy_script) self.assertIn("O_NOFOLLOW", deploy_script) self.assertIn('"max_consecutive_failures": 3', deploy_script) From 2d8f795718791b39c9132dbed0ae0ba136a87e3a Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:20:54 +0800 Subject: [PATCH 23/72] fix: thread requested mode through triage Co-Authored-By: Codex --- service/ai_gateway_service.py | 12 ++++++++++-- service/automation_decision.py | 8 +++++++- tests/test_ai_gateway_service_get_routes.py | 18 ++++++++++++++++++ tests/test_automation_decision.py | 12 +++++++++++- 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 44045b12..f7ddb7ee 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -73,7 +73,7 @@ get_automation_run_ledger, suggest_control_action, ) -from service.automation_decision import EXECUTION_DEFER, EXECUTION_HUMAN_REVIEW, decide_automation_execution, load_execution_policy +from service.automation_decision import EXECUTION_DEFER, EXECUTION_HUMAN_REVIEW, EXECUTION_REVIEW_ONLY, decide_automation_execution, load_execution_policy from service.strategy_automation_registry import ( apply_strategy_registry_guard, summarize_strategy_registry_context, @@ -560,6 +560,8 @@ def _automation_control_snapshot( strict_action = original_action if execution.get("action") == EXECUTION_HUMAN_REVIEW: strict_action = CONTROL_ESCALATE + elif execution.get("action") == EXECUTION_REVIEW_ONLY: + strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX elif ( @@ -602,12 +604,13 @@ def _automation_triage_snapshot( repo: str, *, task: str = "", + requested_mode: str = MODE_REVIEW_ONLY, failure_category: str = "", error: str = "", changed_paths: list[str] | None = None, run_id: str = "", ) -> dict[str, Any]: - control = _automation_control_snapshot(repo, task_name=task) + control = _automation_control_snapshot(repo, task_name=task, requested_mode=requested_mode) policy = load_autonomy_policy() normalized_paths = [ normalized @@ -1524,6 +1527,10 @@ def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, A params = parse_qs(parsed.query) repo = str(payload.get("source_repository") or params.get("repo", [""])[0] or claims.get("repository") or "unknown") task = str(payload.get("task") or params.get("task", [""])[0] or "") + requested_mode = _normalize_control_mode_param(str(payload.get("mode") or params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY)) + if not requested_mode: + _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) + return failure_category = str(payload.get("failure_category") or params.get("failure_category", [""])[0] or "") error = str(payload.get("error") or params.get("error", [""])[0] or "") run_id = str(payload.get("run_id") or params.get("run_id", [""])[0] or "") @@ -1533,6 +1540,7 @@ def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, A triage = _automation_triage_snapshot( repo, task=task, + requested_mode=requested_mode, failure_category=failure_category, error=error, changed_paths=[str(item) for item in changed_paths_raw if isinstance(item, str)], diff --git a/service/automation_decision.py b/service/automation_decision.py index 45fdafe7..cc16ef6b 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -78,6 +78,8 @@ def _normalize_requested_autonomy(value: str) -> str: return AUTONOMY_AUTO_MERGE if mode in {MODE_REVIEW_AND_FIX, AUTONOMY_AUTO_PR}: return AUTONOMY_AUTO_PR + if mode == AUTONOMY_MANUAL: + return AUTONOMY_MANUAL return AUTONOMY_REVIEW_ONLY @@ -360,9 +362,13 @@ def decide_automation_execution( reasons.append(autonomy_config_error) if policy_load_error: reasons.append(policy_load_error) - if effective_autonomy == AUTONOMY_MANUAL: + if max_autonomy == AUTONOMY_MANUAL: action = EXECUTION_HUMAN_REVIEW human_review_required = True + elif requested_autonomy == AUTONOMY_MANUAL: + action = EXECUTION_REVIEW_ONLY + effective_mode = MODE_REVIEW_ONLY + reasons.append("manual mode requested; forcing review_only") if failures >= max_failures: action = EXECUTION_HUMAN_REVIEW diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 6507ac4a..af014d18 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -607,10 +607,26 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: def test_automation_triage_reports_retryable_incident(self) -> None: with tempfile.TemporaryDirectory() as tmp: + policy_path = os.path.join(tmp, "execution_policy.json") + with open(policy_path, "w", encoding="utf-8") as handle: + json.dump( + { + "default": { + "max_autonomy": "auto_pr", + "max_consecutive_failures": 3, + "low_cost_model": "gpt-5.4-mini", + "low_cost_provider": "openai", + }, + "repositories": {}, + }, + handle, + ) env = { "CODEX_AUDIT_SERVICE_AUTH": "none", "CODEX_AUDIT_SERVICE_ALLOW_NO_AUTH_FOR_LOCAL_TESTS": "true", "CODEX_AUDIT_SERVICE_JOB_DIR": tmp, + "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH": policy_path, + "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_OWNER": f"{os.getuid()}:{os.getgid()}", "CODEX_AUDIT_SERVICE_AUTOMATION_OPERATOR_REPOSITORIES": "local", } with patch.dict(os.environ, env, clear=False): @@ -626,6 +642,7 @@ def test_automation_triage_reports_retryable_incident(self) -> None: "error": "codex service timed out waiting for a background job", "changed_paths": ["docs/runbook.md"], "run_id": "incident-123", + "mode": "manual", } request = urllib.request.Request( f"{base_url}/v1/ai/automation/triage", @@ -643,6 +660,7 @@ def test_automation_triage_reports_retryable_incident(self) -> None: self.assertTrue(triage["retry_allowed"]) self.assertEqual(triage["recommended_action"], "retry") self.assertEqual(triage["file_risk"], "low") + self.assertEqual(triage["control"]["execution"]["action"], "review_only") self.assertIn("run_id=incident-123", triage["summary"]) finally: server.shutdown() diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 25bd927b..4d4bc5ad 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -12,6 +12,7 @@ from service.automation_decision import ( EXECUTION_DEFER, EXECUTION_HUMAN_REVIEW, + EXECUTION_REVIEW_ONLY, EXECUTION_RUN, MODE_REVIEW_AND_FIX, MODE_REVIEW_ONLY, @@ -39,7 +40,7 @@ def test_healthy_control_allows_review_and_fix(self) -> None: def test_legacy_autonomy_modes_are_normalized(self) -> None: cases = { - "manual": (MODE_REVIEW_ONLY, "review_only", "review_only", False), + "manual": (MODE_REVIEW_ONLY, "manual", "manual", False), "auto_pr": (MODE_REVIEW_AND_FIX, "auto_pr", "auto_pr", False), "auto_merge": (MODE_REVIEW_AND_FIX, "auto_merge", "auto_pr", False), } @@ -57,6 +58,15 @@ def test_legacy_autonomy_modes_are_normalized(self) -> None: self.assertEqual(result["effective_autonomy"], effective_autonomy) self.assertEqual(result["auto_merge_allowed"], auto_merge_allowed) self.assertFalse(result["human_review_required"]) + manual_result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode="manual", + control_action=CONTROL_CONTINUE, + service_health="healthy", + quota_status="ok", + org_health_status="ok", + ) + self.assertEqual(manual_result["action"], EXECUTION_REVIEW_ONLY) def test_auto_merge_requires_matching_repo_autonomy(self) -> None: result = decide_automation_execution( From 29d221e86f4f3f9a01b8185b7f1af679831c46c5 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:28:28 +0800 Subject: [PATCH 24/72] fix: avoid enforcing truncated failure streaks Co-Authored-By: Codex --- service/ai_gateway_service.py | 10 ++++++- service/automation_decision.py | 15 ++++++++++- tests/test_ai_gateway_automation_control.py | 5 ++-- tests/test_automation_decision.py | 29 +++++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index f7ddb7ee..f06508c5 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -526,10 +526,16 @@ def _automation_control_snapshot( quota_status = {"status": "unavailable"} control = suggest_control_action(get_health_monitor().status, quota_status, org_health) try: - recent_runs = get_automation_run_ledger().snapshot(limit=None)["runs"] + ledger_snapshot = get_automation_run_ledger().snapshot(limit=None) + recent_runs = ledger_snapshot["runs"] + ledger_summary = ledger_snapshot.get("summary") if isinstance(ledger_snapshot.get("summary"), dict) else {} + retention = ledger_summary.get("retention") if isinstance(ledger_summary.get("retention"), dict) else {} + max_retained_runs = int(retention.get("max_runs") or 0) + failure_history_complete = max_retained_runs <= 0 or int(ledger_summary.get("total_runs") or 0) < max_retained_runs ledger_unavailable = False except Exception: recent_runs = [] + failure_history_complete = False ledger_unavailable = True if pending_run is not None: pending_run_id = str(pending_run.get("run_id") or "") @@ -545,6 +551,7 @@ def _automation_control_snapshot( quota_status=quota_status, org_health_status=control.get("org_health_status"), recent_runs=recent_runs, + failure_history_complete=failure_history_complete, policy=load_execution_policy(), ) if ledger_unavailable: @@ -552,6 +559,7 @@ def _automation_control_snapshot( execution["effective_mode"] = MODE_REVIEW_ONLY execution["human_review_required"] = True execution["auto_fix_allowed"] = False + execution["auto_merge_allowed"] = False execution["defer"] = False reasons = execution.get("reasons") if isinstance(execution.get("reasons"), list) else [] reasons.append("automation ledger unavailable; forcing human review") diff --git a/service/automation_decision.py b/service/automation_decision.py index cc16ef6b..3b57e4df 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -290,9 +290,11 @@ def consecutive_failure_count( runs: list[dict[str, Any]], *, repo: str, + require_terminal_boundary: bool = False, ) -> int: """Count latest consecutive trusted failed runs for one repo from newest-first runs.""" count = 0 + terminal_boundary_seen = False normalized_repo = _normalize_repo_id(repo) for run in runs: if not isinstance(run, dict): @@ -308,7 +310,10 @@ def consecutive_failure_count( continue if state in {"queued", "running", "pending", "in_progress"}: continue + terminal_boundary_seen = True break + if require_terminal_boundary and count > 0 and not terminal_boundary_seen: + return 0 return count @@ -324,6 +329,7 @@ def decide_automation_execution( quota_status: Any = "", org_health_status: Any = "", recent_runs: list[dict[str, Any]] | None = None, + failure_history_complete: bool = True, policy: dict[str, Any] | None = None, ) -> dict[str, Any]: """Produce a safe execution decision from health, quota, failures, and repo policy.""" @@ -353,7 +359,11 @@ def decide_automation_execution( service = _normalize_status(service_health) quota = _normalize_quota_status(quota_status) org_health = _normalize_status(org_health_status) - failures = consecutive_failure_count(recent_runs or [], repo=repo) + failures = consecutive_failure_count( + recent_runs or [], + repo=repo, + require_terminal_boundary=not failure_history_complete, + ) if AUTONOMY_RANK[max_autonomy] <= AUTONOMY_RANK[AUTONOMY_REVIEW_ONLY]: effective_mode = MODE_REVIEW_ONLY @@ -375,6 +385,8 @@ def decide_automation_execution( effective_mode = MODE_REVIEW_ONLY human_review_required = True reasons.append(f"consecutive failures reached {failures}/{max_failures}") + elif not failure_history_complete: + reasons.append("failure history may be truncated; not enforcing failure threshold") if control_action in {CONTROL_REVIEW_ONLY, CONTROL_PAUSE_AUTO_FIX, CONTROL_ESCALATE}: effective_mode = MODE_REVIEW_ONLY @@ -430,6 +442,7 @@ def decide_automation_execution( "max_autonomy": max_autonomy, "consecutive_failures": failures, "max_consecutive_failures": max_failures, + "failure_history_complete": failure_history_complete, "human_review_required": human_review_required, "auto_fix_allowed": action == EXECUTION_RUN and effective_mode == MODE_REVIEW_AND_FIX and not human_review_required, "auto_merge_allowed": action == EXECUTION_RUN and effective_autonomy == AUTONOMY_AUTO_MERGE and not human_review_required, diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index cd8f4c9e..51801497 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -249,12 +249,13 @@ def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: patch("service.ai_gateway_service.get_health_monitor", return_value=health), patch("service.ai_gateway_service.get_quota_manager", return_value=quota), patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), - patch("service.ai_gateway_service.load_execution_policy", return_value={}), + patch("service.ai_gateway_service.load_execution_policy", return_value={"default": {"max_autonomy": "auto_merge"}}), ): - control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") self.assertEqual(control["action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") + self.assertFalse(control["execution"]["auto_merge_allowed"]) if __name__ == "__main__": diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 4d4bc5ad..9d44dbbc 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -252,6 +252,35 @@ def test_consecutive_failures_force_human_review(self) -> None: self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) + def test_truncated_failure_history_does_not_force_human_review(self) -> None: + runs = [ + { + "task_name": "monthly", + "task_state": "failed", + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, + }, + { + "task_name": "runtime-health", + "task_state": "failed", + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, + }, + ] + + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode=MODE_REVIEW_AND_FIX, + control_action=CONTROL_CONTINUE, + service_health="healthy", + quota_status="ok", + org_health_status="ok", + recent_runs=runs, + failure_history_complete=False, + policy={"default": {"max_consecutive_failures": 2}}, + ) + + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge", require_terminal_boundary=True), 0) + self.assertEqual(result["action"], EXECUTION_RUN) + def test_running_state_does_not_clear_failure_streak(self) -> None: runs = [ { From 122be8b4c85107d2e010630dc96c62d91a070f41 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:35:51 +0800 Subject: [PATCH 25/72] fix: downgrade legacy action on autonomy cap Co-Authored-By: Codex --- service/ai_gateway_service.py | 9 +++++++-- tests/test_ai_gateway_automation_control.py | 18 ++++++++++++++++++ tests/test_ai_gateway_service_get_routes.py | 11 +++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index f06508c5..fbfc251b 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -572,6 +572,8 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX + elif execution.get("requested_autonomy") != execution.get("effective_autonomy") and strict_action == CONTROL_CONTINUE: + strict_action = CONTROL_REVIEW_ONLY elif ( execution.get("requested_mode") == MODE_REVIEW_AND_FIX and execution.get("effective_mode") == MODE_REVIEW_ONLY @@ -583,7 +585,7 @@ def _automation_control_snapshot( reasons = control.get("reasons") if isinstance(control.get("reasons"), list) else [] reasons.append("capped by execution decision") control["reasons"] = reasons - control["auto_fix_allowed"] = bool(execution.get("auto_fix_allowed")) + control["auto_fix_allowed"] = bool(execution.get("auto_fix_allowed")) and strict_action == CONTROL_CONTINUE control["requires_human_review"] = bool(execution.get("human_review_required")) control["execution"] = execution return control @@ -1620,6 +1622,9 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st _assert_automation_run_access(existing, claims) task_name = str(payload.get("task") or payload.get("task_name") or "") task_state = str(payload.get("task_state") or payload.get("state") or "running") + requested_mode = _normalize_control_mode_param(str(payload.get("mode") or MODE_REVIEW_ONLY)) + if not requested_mode: + raise ValueError("invalid mode") run_metadata = { **metadata, "origin": "external_workflow", @@ -1630,7 +1635,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st control = _automation_control_snapshot( repo, task_name=task_name, - requested_mode=str(payload.get("mode") or MODE_REVIEW_AND_FIX), + requested_mode=requested_mode, pending_run={ "run_id": run_id, "task_name": task_name, diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 51801497..9d534f81 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -182,6 +182,24 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) + def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(self) -> None: + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={"default": {"max_autonomy": "auto_pr"}}), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") + + self.assertEqual(control["action"], "review_only") + self.assertFalse(control["auto_fix_allowed"]) + self.assertFalse(control["execution"]["auto_merge_allowed"]) + def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: runs = [ { diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index af014d18..eb92c80e 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -587,6 +587,17 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: self.assertEqual(recorded["run"]["quota_status"], recorded["control"]["quota_status"]) self.assertEqual(recorded["run"]["org_health_status"], recorded["control"]["org_health_status"]) + invalid_mode_payload = {**payload, "run_id": "platform-health-run-invalid", "mode": "bad"} + invalid_mode_request = urllib.request.Request( + f"{base_url}/v1/ai/automation/runs", + data=json.dumps(invalid_mode_payload).encode("utf-8"), + method="POST", + headers={"Content-Type": "application/json"}, + ) + with self.assertRaises(urllib.error.HTTPError) as ctx: + urllib.request.urlopen(invalid_mode_request, timeout=5) + self.assertEqual(ctx.exception.code, 400) + with urllib.request.urlopen(f"{base_url}/v1/ai/automation/runs?include_events=true", timeout=5) as response: ledger = json.loads(response.read().decode("utf-8"))["ledger"] self.assertEqual(ledger["summary"]["total_runs"], 1) From d5e1e2755be67e29ad32291f41727185d06e69d9 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:46:11 +0800 Subject: [PATCH 26/72] fix: preserve auto pr execution on merge cap Co-Authored-By: Codex --- scripts/deploy_codex_audit_service.sh | 85 ++++++++++++++++++--- service/ai_gateway_service.py | 2 - tests/test_ai_gateway_automation_control.py | 8 +- tests/test_run_monthly_codex_audit.py | 3 + 4 files changed, 83 insertions(+), 15 deletions(-) diff --git a/scripts/deploy_codex_audit_service.sh b/scripts/deploy_codex_audit_service.sh index cbbe8aff..c3161350 100644 --- a/scripts/deploy_codex_audit_service.sh +++ b/scripts/deploy_codex_audit_service.sh @@ -219,10 +219,6 @@ write_default_execution_policy_if_missing() { local policy_path="${EXECUTION_POLICY_FILE}" local policy_dir policy_dir="$(dirname "$policy_path")" - if [ -L "$policy_dir" ]; then - echo "refusing to write execution policy under symlinked directory: $policy_dir" >&2 - exit 1 - fi if [ -L "$policy_path" ]; then echo "refusing to write execution policy through symlink: $policy_path" >&2 exit 1 @@ -230,12 +226,69 @@ write_default_execution_policy_if_missing() { if [ -e "$policy_path" ]; then return fi - sudo install -d -m 0755 -o root -g root "$policy_dir" sudo python3 - "$policy_path" <<'PY' import os +import stat import sys path = sys.argv[1] +if not os.path.isabs(path): + print(f"refusing to write execution policy to relative path: {path}", file=sys.stderr) + raise SystemExit(1) + +policy_dir, policy_name = os.path.split(path) +if not policy_dir or not policy_name: + print(f"invalid execution policy path: {path}", file=sys.stderr) + raise SystemExit(1) + +flags_dir = os.O_RDONLY | os.O_DIRECTORY +if hasattr(os, "O_NOFOLLOW"): + flags_dir |= os.O_NOFOLLOW + + +def fail(message: str) -> None: + print(message, file=sys.stderr) + raise SystemExit(1) + + +def ensure_trusted_dir(fd: int, label: str, *, created: bool) -> None: + info = os.fstat(fd) + if not stat.S_ISDIR(info.st_mode): + fail(f"execution policy parent path is not a directory: {label}") + if created: + os.fchown(fd, 0, 0) + os.fchmod(fd, 0o755) + info = os.fstat(fd) + if (info.st_uid, info.st_gid) != (0, 0): + fail(f"execution policy parent directory owner is invalid: {label}") + if info.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + fail(f"execution policy parent directory permissions are too broad: {label}") + + +def open_admin_policy_dir(directory: str) -> int: + fd = os.open(os.sep, flags_dir) + ensure_trusted_dir(fd, os.sep, created=False) + for component in [part for part in directory.split(os.sep) if part]: + if component in {".", ".."}: + fail(f"invalid execution policy directory component: {component}") + created = False + try: + next_fd = os.open(component, flags_dir, dir_fd=fd) + except FileNotFoundError: + os.mkdir(component, 0o755, dir_fd=fd) + created = True + next_fd = os.open(component, flags_dir, dir_fd=fd) + except OSError as exc: + fail(f"refusing to write execution policy under unsafe directory component {component}: {exc}") + try: + ensure_trusted_dir(next_fd, component, created=created) + finally: + os.close(fd) + fd = next_fd + return fd + + +policy_dir_fd = open_admin_policy_dir(policy_dir) flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW @@ -250,16 +303,28 @@ content = """{ } """ try: - fd = os.open(path, flags, 0o600) + fd = os.open(policy_name, flags, 0o600, dir_fd=policy_dir_fd) except FileExistsError: if os.path.islink(path): print(f"refusing to write execution policy through symlink: {path}", file=sys.stderr) raise SystemExit(1) raise SystemExit(0) -with os.fdopen(fd, "w", encoding="utf-8") as handle: - handle.write(content) -os.chown(path, 0, 0) -os.chmod(path, 0o644) +finally: + os.close(policy_dir_fd) +try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + fd = -1 + handle.write(content) + handle.flush() + os.fchown(handle.fileno(), 0, 0) + os.fchmod(handle.fileno(), 0o644) +except Exception: + if fd >= 0: + try: + os.close(fd) + except OSError: + pass + raise PY } diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index fbfc251b..9048b0f6 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -572,8 +572,6 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX - elif execution.get("requested_autonomy") != execution.get("effective_autonomy") and strict_action == CONTROL_CONTINUE: - strict_action = CONTROL_REVIEW_ONLY elif ( execution.get("requested_mode") == MODE_REVIEW_AND_FIX and execution.get("effective_mode") == MODE_REVIEW_ONLY diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 9d534f81..7af1c068 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -182,7 +182,7 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) - def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(self) -> None: + def test_control_snapshot_keeps_legacy_continue_when_only_auto_merge_is_capped(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -196,8 +196,10 @@ def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(sel ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") - self.assertEqual(control["action"], "review_only") - self.assertFalse(control["auto_fix_allowed"]) + self.assertEqual(control["action"], "continue") + self.assertTrue(control["auto_fix_allowed"]) + self.assertEqual(control["execution"]["requested_autonomy"], "auto_merge") + self.assertEqual(control["execution"]["effective_autonomy"], "auto_pr") self.assertFalse(control["execution"]["auto_merge_allowed"]) def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: diff --git a/tests/test_run_monthly_codex_audit.py b/tests/test_run_monthly_codex_audit.py index 5f0112a2..faa4387f 100644 --- a/tests/test_run_monthly_codex_audit.py +++ b/tests/test_run_monthly_codex_audit.py @@ -2346,6 +2346,9 @@ def test_vps_deploy_adds_nginx_audit_route_without_router_service(self) -> None: self.assertIn("/etc/codex-audit-bridge-policy/execution_policy.json", deploy_script) self.assertIn("write_default_execution_policy_if_missing", deploy_script) self.assertIn("O_NOFOLLOW", deploy_script) + self.assertIn("os.O_DIRECTORY", deploy_script) + self.assertIn("dir_fd=fd", deploy_script) + self.assertIn("os.open(component, flags_dir, dir_fd=fd)", deploy_script) self.assertIn('"max_consecutive_failures": 3', deploy_script) self.assertIn("codex_pr_review.yml@refs/pull/*/merge", deploy_script) self.assertIn("refs/pull/*/merge", deploy_script) From 10c2917c97c68cce2d5554d5632b2b1048297708 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 05:56:11 +0800 Subject: [PATCH 27/72] fix: enforce retained failure streaks Co-Authored-By: Codex --- service/ai_gateway_service.py | 5 ++- service/automation_decision.py | 8 +--- service/automation_run_ledger.py | 31 +++++++++---- tests/test_ai_gateway_automation_control.py | 50 +++++++++++++++++++-- tests/test_automation_decision.py | 6 +-- tests/test_automation_run_ledger.py | 17 +++++++ 6 files changed, 93 insertions(+), 24 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 9048b0f6..5d6a7000 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -530,8 +530,7 @@ def _automation_control_snapshot( recent_runs = ledger_snapshot["runs"] ledger_summary = ledger_snapshot.get("summary") if isinstance(ledger_snapshot.get("summary"), dict) else {} retention = ledger_summary.get("retention") if isinstance(ledger_summary.get("retention"), dict) else {} - max_retained_runs = int(retention.get("max_runs") or 0) - failure_history_complete = max_retained_runs <= 0 or int(ledger_summary.get("total_runs") or 0) < max_retained_runs + failure_history_complete = not bool(retention.get("may_be_truncated")) ledger_unavailable = False except Exception: recent_runs = [] @@ -572,6 +571,8 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX + elif execution.get("requested_autonomy") != execution.get("effective_autonomy") and strict_action == CONTROL_CONTINUE: + strict_action = CONTROL_REVIEW_ONLY elif ( execution.get("requested_mode") == MODE_REVIEW_AND_FIX and execution.get("effective_mode") == MODE_REVIEW_ONLY diff --git a/service/automation_decision.py b/service/automation_decision.py index 3b57e4df..2d55ad76 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -290,11 +290,9 @@ def consecutive_failure_count( runs: list[dict[str, Any]], *, repo: str, - require_terminal_boundary: bool = False, ) -> int: """Count latest consecutive trusted failed runs for one repo from newest-first runs.""" count = 0 - terminal_boundary_seen = False normalized_repo = _normalize_repo_id(repo) for run in runs: if not isinstance(run, dict): @@ -310,10 +308,7 @@ def consecutive_failure_count( continue if state in {"queued", "running", "pending", "in_progress"}: continue - terminal_boundary_seen = True break - if require_terminal_boundary and count > 0 and not terminal_boundary_seen: - return 0 return count @@ -362,7 +357,6 @@ def decide_automation_execution( failures = consecutive_failure_count( recent_runs or [], repo=repo, - require_terminal_boundary=not failure_history_complete, ) if AUTONOMY_RANK[max_autonomy] <= AUTONOMY_RANK[AUTONOMY_REVIEW_ONLY]: @@ -386,7 +380,7 @@ def decide_automation_execution( human_review_required = True reasons.append(f"consecutive failures reached {failures}/{max_failures}") elif not failure_history_complete: - reasons.append("failure history may be truncated; not enforcing failure threshold") + reasons.append("failure history may be truncated; using retained failure streak") if control_action in {CONTROL_REVIEW_ONLY, CONTROL_PAUSE_AUTO_FIX, CONTROL_ESCALATE}: effective_mode = MODE_REVIEW_ONLY diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 4d1827b6..882b1230 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -214,31 +214,34 @@ def __init__( self._max_runs = max(1, int(max_runs)) self._max_events_per_run = max(1, int(max_events_per_run)) self._sequence = 0 + self._evicted_runs_count = 0 self._storage_path = storage_path self._load_from_disk() def _load_from_disk(self) -> None: if self._storage_path is None or not self._storage_path.exists(): return - runs, sequence = self._read_from_disk_unlocked() + runs, sequence, evicted_runs = self._read_from_disk_unlocked() with self._lock: self._runs = runs self._sequence = sequence + self._evicted_runs_count = evicted_runs self._evict_old_runs_locked() - def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int]: + def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int]: if self._storage_path is None or not self._storage_path.exists(): - return {}, 0 + return {}, 0, 0 try: payload = json.loads(self._storage_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): - return {}, 0 + return {}, 0, 0 runs = payload.get("runs") if isinstance(payload, dict) else None if not isinstance(runs, dict): - return {}, 0 + return {}, 0, 0 clean_runs = {str(key): value for key, value in runs.items() if isinstance(value, dict)} sequence = _safe_int(payload.get("sequence"), len(clean_runs)) - return clean_runs, sequence + evicted_runs = _safe_int(payload.get("evicted_runs"), 0) + return clean_runs, sequence, max(0, evicted_runs) def _persist_locked(self) -> None: self._persist_with_owner_guard_locked() @@ -252,7 +255,7 @@ def _refresh_from_disk_locked(self) -> None: lock_path = self._storage_path.with_suffix(self._storage_path.suffix + ".lock") lock_handle = lock_path.open("a+", encoding="utf-8") fcntl.flock(lock_handle.fileno(), fcntl.LOCK_SH) - disk_runs, disk_sequence = self._read_from_disk_unlocked() + disk_runs, disk_sequence, disk_evicted_runs = self._read_from_disk_unlocked() for run_id, disk_entry in disk_runs.items(): current = self._runs.get(run_id) if current is not None: @@ -260,6 +263,7 @@ def _refresh_from_disk_locked(self) -> None: else: self._runs[run_id] = disk_entry self._sequence = max(self._sequence, disk_sequence, len(self._runs)) + self._evicted_runs_count = max(self._evicted_runs_count, disk_evicted_runs) self._evict_old_runs_locked() finally: if lock_handle is not None: @@ -283,7 +287,7 @@ def _persist_with_owner_guard_locked( lock_path = self._storage_path.with_suffix(self._storage_path.suffix + ".lock") lock_handle = lock_path.open("a+", encoding="utf-8") fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) - disk_runs, disk_sequence = self._read_from_disk_unlocked() + disk_runs, disk_sequence, disk_evicted_runs = self._read_from_disk_unlocked() if guard_run_id and owner_repository: disk_entry = disk_runs.get(guard_run_id) disk_owner = _entry_owner_repository(disk_entry) if isinstance(disk_entry, dict) else "" @@ -296,6 +300,7 @@ def _persist_with_owner_guard_locked( else: self._runs[run_id] = disk_entry self._sequence = max(self._sequence, disk_sequence, len(self._runs)) + self._evicted_runs_count = max(self._evicted_runs_count, disk_evicted_runs) self._evict_old_runs_locked() self._write_to_disk_locked() finally: @@ -311,6 +316,7 @@ def _write_to_disk_locked(self) -> None: payload = { "schema_version": "automation_run_ledger.v1", "sequence": self._sequence, + "evicted_runs": self._evicted_runs_count, "runs": self._runs, } tmp = self._storage_path.with_suffix(self._storage_path.suffix + ".tmp") @@ -366,6 +372,7 @@ def _evict_old_runs_locked(self) -> None: ) for entry in ordered[:overflow]: self._runs.pop(str(entry["run_id"])) + self._evicted_runs_count += overflow @staticmethod def _public_entry(entry: dict[str, Any], *, include_events: bool = True) -> dict[str, Any]: @@ -407,6 +414,7 @@ def record( with self._lock: previous_runs = deepcopy(self._runs) previous_sequence = self._sequence + previous_evicted_runs_count = self._evicted_runs_count current = self._runs.get(run_id) if current: current_owner = _entry_owner_repository(current) @@ -451,12 +459,14 @@ def record( } ) self._runs[run_id] = entry - self._evict_old_runs_locked() + if self._storage_path is None: + self._evict_old_runs_locked() try: self._persist_with_owner_guard_locked(guard_run_id=run_id, owner_repository=owner_repository) except Exception: self._runs = previous_runs self._sequence = previous_sequence + self._evicted_runs_count = previous_evicted_runs_count raise return self._public_entry(self._runs[run_id]) @@ -476,6 +486,7 @@ def snapshot(self, *, limit: int | None = 100, include_events: bool = False) -> ] max_runs = self._max_runs max_events_per_run = self._max_events_per_run + evicted_runs_count = self._evicted_runs_count task_states = Counter(str(run.get("task_state", "")).strip().lower() for run in retained_runs if run.get("task_state")) suggested_actions = Counter( @@ -505,6 +516,8 @@ def snapshot(self, *, limit: int | None = 100, include_events: bool = False) -> "max_runs": max_runs, "max_events_per_run": max_events_per_run, "events_included": include_events, + "evicted_runs": evicted_runs_count, + "may_be_truncated": evicted_runs_count > 0, }, }, } diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 7af1c068..f91d1e67 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -182,7 +182,49 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) - def test_control_snapshot_keeps_legacy_continue_when_only_auto_merge_is_capped(self) -> None: + def test_control_snapshot_enforces_retained_failures_after_ledger_eviction(self) -> None: + runs = [ + { + "run_id": "failed-2", + "task_name": "monthly", + "task_state": "failed", + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}, + }, + { + "run_id": "failed-1", + "task_name": "monthly", + "task_state": "failed", + "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}, + }, + ] + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type( + "Ledger", + (), + { + "snapshot": lambda self, limit=None: { + "runs": runs, + "summary": {"retention": {"may_be_truncated": True, "evicted_runs": 1}}, + } + }, + )() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={"default": {"max_consecutive_failures": 2}}), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") + + self.assertEqual(control["action"], "escalate") + self.assertEqual(control["execution"]["action"], "human_review") + self.assertFalse(control["execution"]["failure_history_complete"]) + self.assertEqual(control["execution"]["consecutive_failures"], 2) + + def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -196,10 +238,12 @@ def test_control_snapshot_keeps_legacy_continue_when_only_auto_merge_is_capped(s ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") - self.assertEqual(control["action"], "continue") - self.assertTrue(control["auto_fix_allowed"]) + self.assertEqual(control["action"], "review_only") + self.assertFalse(control["auto_fix_allowed"]) self.assertEqual(control["execution"]["requested_autonomy"], "auto_merge") self.assertEqual(control["execution"]["effective_autonomy"], "auto_pr") + self.assertEqual(control["execution"]["action"], "run") + self.assertTrue(control["execution"]["auto_fix_allowed"]) self.assertFalse(control["execution"]["auto_merge_allowed"]) def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 9d44dbbc..81289aab 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -252,7 +252,7 @@ def test_consecutive_failures_force_human_review(self) -> None: self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) - def test_truncated_failure_history_does_not_force_human_review(self) -> None: + def test_truncated_failure_history_uses_retained_failure_streak(self) -> None: runs = [ { "task_name": "monthly", @@ -278,8 +278,8 @@ def test_truncated_failure_history_does_not_force_human_review(self) -> None: policy={"default": {"max_consecutive_failures": 2}}, ) - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge", require_terminal_boundary=True), 0) - self.assertEqual(result["action"], EXECUTION_RUN) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 2) + self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) def test_running_state_does_not_clear_failure_streak(self) -> None: runs = [ diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index e4ca5655..2d33f8e6 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -175,6 +175,8 @@ def test_ledger_evicts_old_runs_by_count(self) -> None: snapshot = ledger.snapshot(limit=None) self.assertEqual(snapshot["summary"]["total_runs"], 2) + self.assertEqual(snapshot["summary"]["retention"]["evicted_runs"], 1) + self.assertTrue(snapshot["summary"]["retention"]["may_be_truncated"]) self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-2", "run-3"}) def test_ledger_eviction_keeps_new_run_when_timestamps_match(self) -> None: @@ -186,6 +188,21 @@ def test_ledger_eviction_keeps_new_run_when_timestamps_match(self) -> None: snapshot = ledger.snapshot(limit=None) self.assertEqual([run["run_id"] for run in snapshot["runs"]], ["run-2"]) + def test_persisted_ledger_eviction_count_is_not_double_counted(self) -> None: + with TemporaryDirectory() as tmp: + path = Path(tmp) / "automation_runs.json" + ledger = AutomationRunLedger(max_runs=2, storage_path=path) + ledger.record("run-1", "queued") + ledger.record("run-2", "queued") + ledger.record("run-3", "queued") + + reloaded = AutomationRunLedger(max_runs=2, storage_path=path) + snapshot = reloaded.snapshot(limit=None) + + self.assertEqual(snapshot["summary"]["retention"]["evicted_runs"], 1) + self.assertTrue(snapshot["summary"]["retention"]["may_be_truncated"]) + self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-2", "run-3"}) + def test_update_preserves_control_fields_when_omitted(self) -> None: self.ledger.record( "run-1", From 259cde84734b74d703190f31e1a50e7f1d9474c8 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 06:02:00 +0800 Subject: [PATCH 28/72] fix: fail closed on truncated failure history Co-Authored-By: Codex --- service/ai_gateway_service.py | 3 +-- service/automation_decision.py | 5 ++++- tests/test_ai_gateway_automation_control.py | 17 ++++++----------- tests/test_automation_decision.py | 9 ++------- 4 files changed, 13 insertions(+), 21 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 5d6a7000..8cd8d11d 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -571,8 +571,6 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX - elif execution.get("requested_autonomy") != execution.get("effective_autonomy") and strict_action == CONTROL_CONTINUE: - strict_action = CONTROL_REVIEW_ONLY elif ( execution.get("requested_mode") == MODE_REVIEW_AND_FIX and execution.get("effective_mode") == MODE_REVIEW_ONLY @@ -585,6 +583,7 @@ def _automation_control_snapshot( reasons.append("capped by execution decision") control["reasons"] = reasons control["auto_fix_allowed"] = bool(execution.get("auto_fix_allowed")) and strict_action == CONTROL_CONTINUE + control["auto_merge_allowed"] = bool(execution.get("auto_merge_allowed")) and strict_action == CONTROL_CONTINUE control["requires_human_review"] = bool(execution.get("human_review_required")) control["execution"] = execution return control diff --git a/service/automation_decision.py b/service/automation_decision.py index 2d55ad76..fcab40b7 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -380,7 +380,10 @@ def decide_automation_execution( human_review_required = True reasons.append(f"consecutive failures reached {failures}/{max_failures}") elif not failure_history_complete: - reasons.append("failure history may be truncated; using retained failure streak") + action = EXECUTION_HUMAN_REVIEW + effective_mode = MODE_REVIEW_ONLY + human_review_required = True + reasons.append("failure history may be truncated; forcing human review") if control_action in {CONTROL_REVIEW_ONLY, CONTROL_PAUSE_AUTO_FIX, CONTROL_ESCALATE}: effective_mode = MODE_REVIEW_ONLY diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index f91d1e67..5a991c58 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -182,14 +182,8 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) - def test_control_snapshot_enforces_retained_failures_after_ledger_eviction(self) -> None: + def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: runs = [ - { - "run_id": "failed-2", - "task_name": "monthly", - "task_state": "failed", - "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}, - }, { "run_id": "failed-1", "task_name": "monthly", @@ -222,9 +216,9 @@ def test_control_snapshot_enforces_retained_failures_after_ledger_eviction(self) self.assertEqual(control["action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertFalse(control["execution"]["failure_history_complete"]) - self.assertEqual(control["execution"]["consecutive_failures"], 2) + self.assertEqual(control["execution"]["consecutive_failures"], 1) - def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(self) -> None: + def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -238,8 +232,9 @@ def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(sel ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") - self.assertEqual(control["action"], "review_only") - self.assertFalse(control["auto_fix_allowed"]) + self.assertEqual(control["action"], "continue") + self.assertTrue(control["auto_fix_allowed"]) + self.assertFalse(control["auto_merge_allowed"]) self.assertEqual(control["execution"]["requested_autonomy"], "auto_merge") self.assertEqual(control["execution"]["effective_autonomy"], "auto_pr") self.assertEqual(control["execution"]["action"], "run") diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 81289aab..2874d412 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -252,18 +252,13 @@ def test_consecutive_failures_force_human_review(self) -> None: self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) - def test_truncated_failure_history_uses_retained_failure_streak(self) -> None: + def test_truncated_failure_history_fails_closed(self) -> None: runs = [ { "task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, }, - { - "task_name": "runtime-health", - "task_state": "failed", - "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, - }, ] result = decide_automation_execution( @@ -278,7 +273,7 @@ def test_truncated_failure_history_uses_retained_failure_streak(self) -> None: policy={"default": {"max_consecutive_failures": 2}}, ) - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 2) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) self.assertEqual(result["action"], EXECUTION_HUMAN_REVIEW) def test_running_state_does_not_clear_failure_streak(self) -> None: From 59b7dd1956331d0fca767a482cb0cc663dca3417 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 06:08:43 +0800 Subject: [PATCH 29/72] fix: scope ledger truncation by repo Co-Authored-By: Codex --- service/ai_gateway_service.py | 11 ++++- service/automation_run_ledger.py | 45 +++++++++++++++++---- tests/test_ai_gateway_automation_control.py | 40 +++++++++++++++++- tests/test_ai_gateway_service_get_routes.py | 1 + tests/test_automation_run_ledger.py | 14 ++++--- 5 files changed, 94 insertions(+), 17 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 8cd8d11d..02af0fac 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -530,7 +530,14 @@ def _automation_control_snapshot( recent_runs = ledger_snapshot["runs"] ledger_summary = ledger_snapshot.get("summary") if isinstance(ledger_snapshot.get("summary"), dict) else {} retention = ledger_summary.get("retention") if isinstance(ledger_summary.get("retention"), dict) else {} - failure_history_complete = not bool(retention.get("may_be_truncated")) + evicted_by_repo = ( + retention.get("evicted_runs_by_repo") if isinstance(retention.get("evicted_runs_by_repo"), dict) else {} + ) + try: + repo_evictions = int(evicted_by_repo.get(str(repo or "unknown").strip().lower(), 0) or 0) + except (TypeError, ValueError): + repo_evictions = 0 + failure_history_complete = repo_evictions <= 0 ledger_unavailable = False except Exception: recent_runs = [] @@ -1620,7 +1627,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st _assert_automation_run_access(existing, claims) task_name = str(payload.get("task") or payload.get("task_name") or "") task_state = str(payload.get("task_state") or payload.get("state") or "running") - requested_mode = _normalize_control_mode_param(str(payload.get("mode") or MODE_REVIEW_ONLY)) + requested_mode = _normalize_control_mode_param(str(payload.get("mode") or MODE_REVIEW_AND_FIX)) if not requested_mode: raise ValueError("invalid mode") run_metadata = { diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 882b1230..e1bd387e 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -132,6 +132,10 @@ def _entry_owner_repository(entry: dict[str, Any]) -> str: return str(metadata.get("source_repository") or metadata.get("repository") or "") +def _repo_retention_key(value: Any) -> str: + return str(value or "").strip().lower() + + def _entry_origin(entry: dict[str, Any]) -> str: metadata = entry.get("metadata") if isinstance(entry.get("metadata"), dict) else {} return str(metadata.get("origin") or "") @@ -215,33 +219,48 @@ def __init__( self._max_events_per_run = max(1, int(max_events_per_run)) self._sequence = 0 self._evicted_runs_count = 0 + self._evicted_runs_by_repo: dict[str, int] = {} self._storage_path = storage_path self._load_from_disk() def _load_from_disk(self) -> None: if self._storage_path is None or not self._storage_path.exists(): return - runs, sequence, evicted_runs = self._read_from_disk_unlocked() + runs, sequence, evicted_runs, evicted_runs_by_repo = self._read_from_disk_unlocked() with self._lock: self._runs = runs self._sequence = sequence self._evicted_runs_count = evicted_runs + self._evicted_runs_by_repo = evicted_runs_by_repo self._evict_old_runs_locked() - def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int]: + def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, dict[str, int]]: if self._storage_path is None or not self._storage_path.exists(): - return {}, 0, 0 + return {}, 0, 0, {} try: payload = json.loads(self._storage_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): - return {}, 0, 0 + return {}, 0, 0, {} runs = payload.get("runs") if isinstance(payload, dict) else None if not isinstance(runs, dict): - return {}, 0, 0 + return {}, 0, 0, {} clean_runs = {str(key): value for key, value in runs.items() if isinstance(value, dict)} sequence = _safe_int(payload.get("sequence"), len(clean_runs)) evicted_runs = _safe_int(payload.get("evicted_runs"), 0) - return clean_runs, sequence, max(0, evicted_runs) + raw_evicted_by_repo = payload.get("evicted_runs_by_repo") + evicted_by_repo = { + _repo_retention_key(repo): max(0, _safe_int(count)) + for repo, count in (raw_evicted_by_repo.items() if isinstance(raw_evicted_by_repo, dict) else []) + if _repo_retention_key(repo) + } + return clean_runs, sequence, max(0, evicted_runs), evicted_by_repo + + def _merge_evicted_runs_by_repo_locked(self, disk_evicted_runs_by_repo: dict[str, int]) -> None: + for repo, count in disk_evicted_runs_by_repo.items(): + repo_key = _repo_retention_key(repo) + if not repo_key: + continue + self._evicted_runs_by_repo[repo_key] = max(self._evicted_runs_by_repo.get(repo_key, 0), int(count)) def _persist_locked(self) -> None: self._persist_with_owner_guard_locked() @@ -255,7 +274,7 @@ def _refresh_from_disk_locked(self) -> None: lock_path = self._storage_path.with_suffix(self._storage_path.suffix + ".lock") lock_handle = lock_path.open("a+", encoding="utf-8") fcntl.flock(lock_handle.fileno(), fcntl.LOCK_SH) - disk_runs, disk_sequence, disk_evicted_runs = self._read_from_disk_unlocked() + disk_runs, disk_sequence, disk_evicted_runs, disk_evicted_runs_by_repo = self._read_from_disk_unlocked() for run_id, disk_entry in disk_runs.items(): current = self._runs.get(run_id) if current is not None: @@ -264,6 +283,7 @@ def _refresh_from_disk_locked(self) -> None: self._runs[run_id] = disk_entry self._sequence = max(self._sequence, disk_sequence, len(self._runs)) self._evicted_runs_count = max(self._evicted_runs_count, disk_evicted_runs) + self._merge_evicted_runs_by_repo_locked(disk_evicted_runs_by_repo) self._evict_old_runs_locked() finally: if lock_handle is not None: @@ -287,7 +307,7 @@ def _persist_with_owner_guard_locked( lock_path = self._storage_path.with_suffix(self._storage_path.suffix + ".lock") lock_handle = lock_path.open("a+", encoding="utf-8") fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) - disk_runs, disk_sequence, disk_evicted_runs = self._read_from_disk_unlocked() + disk_runs, disk_sequence, disk_evicted_runs, disk_evicted_runs_by_repo = self._read_from_disk_unlocked() if guard_run_id and owner_repository: disk_entry = disk_runs.get(guard_run_id) disk_owner = _entry_owner_repository(disk_entry) if isinstance(disk_entry, dict) else "" @@ -301,6 +321,7 @@ def _persist_with_owner_guard_locked( self._runs[run_id] = disk_entry self._sequence = max(self._sequence, disk_sequence, len(self._runs)) self._evicted_runs_count = max(self._evicted_runs_count, disk_evicted_runs) + self._merge_evicted_runs_by_repo_locked(disk_evicted_runs_by_repo) self._evict_old_runs_locked() self._write_to_disk_locked() finally: @@ -317,6 +338,7 @@ def _write_to_disk_locked(self) -> None: "schema_version": "automation_run_ledger.v1", "sequence": self._sequence, "evicted_runs": self._evicted_runs_count, + "evicted_runs_by_repo": self._evicted_runs_by_repo, "runs": self._runs, } tmp = self._storage_path.with_suffix(self._storage_path.suffix + ".tmp") @@ -371,6 +393,9 @@ def _evict_old_runs_locked(self) -> None: ), ) for entry in ordered[:overflow]: + repo_key = _repo_retention_key(_entry_owner_repository(entry)) + if repo_key: + self._evicted_runs_by_repo[repo_key] = self._evicted_runs_by_repo.get(repo_key, 0) + 1 self._runs.pop(str(entry["run_id"])) self._evicted_runs_count += overflow @@ -415,6 +440,7 @@ def record( previous_runs = deepcopy(self._runs) previous_sequence = self._sequence previous_evicted_runs_count = self._evicted_runs_count + previous_evicted_runs_by_repo = dict(self._evicted_runs_by_repo) current = self._runs.get(run_id) if current: current_owner = _entry_owner_repository(current) @@ -467,6 +493,7 @@ def record( self._runs = previous_runs self._sequence = previous_sequence self._evicted_runs_count = previous_evicted_runs_count + self._evicted_runs_by_repo = previous_evicted_runs_by_repo raise return self._public_entry(self._runs[run_id]) @@ -487,6 +514,7 @@ def snapshot(self, *, limit: int | None = 100, include_events: bool = False) -> max_runs = self._max_runs max_events_per_run = self._max_events_per_run evicted_runs_count = self._evicted_runs_count + evicted_runs_by_repo = dict(self._evicted_runs_by_repo) task_states = Counter(str(run.get("task_state", "")).strip().lower() for run in retained_runs if run.get("task_state")) suggested_actions = Counter( @@ -517,6 +545,7 @@ def snapshot(self, *, limit: int | None = 100, include_events: bool = False) -> "max_events_per_run": max_events_per_run, "events_included": include_events, "evicted_runs": evicted_runs_count, + "evicted_runs_by_repo": evicted_runs_by_repo, "may_be_truncated": evicted_runs_count > 0, }, }, diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 5a991c58..ba627c5b 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -199,7 +199,13 @@ def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: { "snapshot": lambda self, limit=None: { "runs": runs, - "summary": {"retention": {"may_be_truncated": True, "evicted_runs": 1}}, + "summary": { + "retention": { + "may_be_truncated": True, + "evicted_runs": 1, + "evicted_runs_by_repo": {"quantstrategylab/targetrepo": 1}, + } + }, } }, )() @@ -218,6 +224,38 @@ def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: self.assertFalse(control["execution"]["failure_history_complete"]) self.assertEqual(control["execution"]["consecutive_failures"], 1) + def test_control_snapshot_ignores_other_repo_ledger_eviction(self) -> None: + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type( + "Ledger", + (), + { + "snapshot": lambda self, limit=None: { + "runs": [], + "summary": { + "retention": { + "may_be_truncated": True, + "evicted_runs": 1, + "evicted_runs_by_repo": {"quantstrategylab/otherrepo": 1}, + } + }, + } + }, + )() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={"default": {"max_consecutive_failures": 2}}), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") + + self.assertEqual(control["action"], "continue") + self.assertTrue(control["execution"]["failure_history_complete"]) + def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index eb92c80e..847940f9 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -583,6 +583,7 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: recorded = json.loads(response.read().decode("utf-8")) self.assertEqual(recorded["run"]["run_id"], "platform-health-run-1") self.assertEqual(recorded["run"]["suggested_action"], recorded["control"]["action"]) + self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_and_fix") self.assertEqual(recorded["run"]["service_health"], recorded["control"]["service_health"]) self.assertEqual(recorded["run"]["quota_status"], recorded["control"]["quota_status"]) self.assertEqual(recorded["run"]["org_health_status"], recorded["control"]["org_health_status"]) diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index 2d33f8e6..e57b53c7 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -169,13 +169,14 @@ def test_snapshot_can_include_bounded_history(self) -> None: def test_ledger_evicts_old_runs_by_count(self) -> None: ledger = AutomationRunLedger(max_runs=2) - ledger.record("run-1", "queued") - ledger.record("run-2", "queued") - ledger.record("run-3", "queued") + ledger.record("run-1", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + ledger.record("run-2", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + ledger.record("run-3", "queued", metadata={"source_repository": "QuantStrategyLab/RepoB"}) snapshot = ledger.snapshot(limit=None) self.assertEqual(snapshot["summary"]["total_runs"], 2) self.assertEqual(snapshot["summary"]["retention"]["evicted_runs"], 1) + self.assertEqual(snapshot["summary"]["retention"]["evicted_runs_by_repo"], {"quantstrategylab/repoa": 1}) self.assertTrue(snapshot["summary"]["retention"]["may_be_truncated"]) self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-2", "run-3"}) @@ -192,14 +193,15 @@ def test_persisted_ledger_eviction_count_is_not_double_counted(self) -> None: with TemporaryDirectory() as tmp: path = Path(tmp) / "automation_runs.json" ledger = AutomationRunLedger(max_runs=2, storage_path=path) - ledger.record("run-1", "queued") - ledger.record("run-2", "queued") - ledger.record("run-3", "queued") + ledger.record("run-1", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + ledger.record("run-2", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + ledger.record("run-3", "queued", metadata={"source_repository": "QuantStrategyLab/RepoB"}) reloaded = AutomationRunLedger(max_runs=2, storage_path=path) snapshot = reloaded.snapshot(limit=None) self.assertEqual(snapshot["summary"]["retention"]["evicted_runs"], 1) + self.assertEqual(snapshot["summary"]["retention"]["evicted_runs_by_repo"], {"quantstrategylab/repoa": 1}) self.assertTrue(snapshot["summary"]["retention"]["may_be_truncated"]) self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-2", "run-3"}) From e381ee93cc12446378886e035ae71f9ff342c641 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 06:16:32 +0800 Subject: [PATCH 30/72] fix: mark legacy ledger history unknown Co-Authored-By: Codex --- service/ai_gateway_service.py | 2 +- service/automation_run_ledger.py | 49 +++++++++++++++++---- tests/test_ai_gateway_automation_control.py | 26 +++++++++++ tests/test_automation_run_ledger.py | 43 ++++++++++++++++++ 4 files changed, 111 insertions(+), 9 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 02af0fac..4ad429e5 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -537,7 +537,7 @@ def _automation_control_snapshot( repo_evictions = int(evicted_by_repo.get(str(repo or "unknown").strip().lower(), 0) or 0) except (TypeError, ValueError): repo_evictions = 0 - failure_history_complete = repo_evictions <= 0 + failure_history_complete = repo_evictions <= 0 and not bool(retention.get("history_completeness_unknown")) ledger_unavailable = False except Exception: recent_runs = [] diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index e1bd387e..d68f56fb 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -220,32 +220,35 @@ def __init__( self._sequence = 0 self._evicted_runs_count = 0 self._evicted_runs_by_repo: dict[str, int] = {} + self._history_completeness_unknown = False self._storage_path = storage_path self._load_from_disk() def _load_from_disk(self) -> None: if self._storage_path is None or not self._storage_path.exists(): return - runs, sequence, evicted_runs, evicted_runs_by_repo = self._read_from_disk_unlocked() + runs, sequence, evicted_runs, evicted_runs_by_repo, history_unknown = self._read_from_disk_unlocked() with self._lock: self._runs = runs self._sequence = sequence self._evicted_runs_count = evicted_runs self._evicted_runs_by_repo = evicted_runs_by_repo + self._history_completeness_unknown = history_unknown self._evict_old_runs_locked() - def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, dict[str, int]]: + def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, dict[str, int], bool]: if self._storage_path is None or not self._storage_path.exists(): - return {}, 0, 0, {} + return {}, 0, 0, {}, False try: payload = json.loads(self._storage_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): - return {}, 0, 0, {} + return {}, 0, 0, {}, False runs = payload.get("runs") if isinstance(payload, dict) else None if not isinstance(runs, dict): - return {}, 0, 0, {} + return {}, 0, 0, {}, False clean_runs = {str(key): value for key, value in runs.items() if isinstance(value, dict)} sequence = _safe_int(payload.get("sequence"), len(clean_runs)) + history_unknown = "evicted_runs" not in payload or "evicted_runs_by_repo" not in payload evicted_runs = _safe_int(payload.get("evicted_runs"), 0) raw_evicted_by_repo = payload.get("evicted_runs_by_repo") evicted_by_repo = { @@ -253,7 +256,8 @@ def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, for repo, count in (raw_evicted_by_repo.items() if isinstance(raw_evicted_by_repo, dict) else []) if _repo_retention_key(repo) } - return clean_runs, sequence, max(0, evicted_runs), evicted_by_repo + history_unknown = bool(payload.get("history_completeness_unknown", history_unknown)) + return clean_runs, sequence, max(0, evicted_runs), evicted_by_repo, history_unknown def _merge_evicted_runs_by_repo_locked(self, disk_evicted_runs_by_repo: dict[str, int]) -> None: for repo, count in disk_evicted_runs_by_repo.items(): @@ -262,6 +266,14 @@ def _merge_evicted_runs_by_repo_locked(self, disk_evicted_runs_by_repo: dict[str continue self._evicted_runs_by_repo[repo_key] = max(self._evicted_runs_by_repo.get(repo_key, 0), int(count)) + def _drop_runs_evicted_on_disk_locked(self, disk_runs: dict[str, dict[str, Any]], *, preserve_run_id: str = "") -> None: + if len(disk_runs) < self._max_runs: + return + disk_run_ids = set(disk_runs) + for run_id in list(self._runs): + if run_id != preserve_run_id and run_id not in disk_run_ids: + self._runs.pop(run_id, None) + def _persist_locked(self) -> None: self._persist_with_owner_guard_locked() @@ -274,7 +286,14 @@ def _refresh_from_disk_locked(self) -> None: lock_path = self._storage_path.with_suffix(self._storage_path.suffix + ".lock") lock_handle = lock_path.open("a+", encoding="utf-8") fcntl.flock(lock_handle.fileno(), fcntl.LOCK_SH) - disk_runs, disk_sequence, disk_evicted_runs, disk_evicted_runs_by_repo = self._read_from_disk_unlocked() + ( + disk_runs, + disk_sequence, + disk_evicted_runs, + disk_evicted_runs_by_repo, + disk_history_unknown, + ) = self._read_from_disk_unlocked() + self._drop_runs_evicted_on_disk_locked(disk_runs) for run_id, disk_entry in disk_runs.items(): current = self._runs.get(run_id) if current is not None: @@ -284,6 +303,7 @@ def _refresh_from_disk_locked(self) -> None: self._sequence = max(self._sequence, disk_sequence, len(self._runs)) self._evicted_runs_count = max(self._evicted_runs_count, disk_evicted_runs) self._merge_evicted_runs_by_repo_locked(disk_evicted_runs_by_repo) + self._history_completeness_unknown = self._history_completeness_unknown or disk_history_unknown self._evict_old_runs_locked() finally: if lock_handle is not None: @@ -307,12 +327,19 @@ def _persist_with_owner_guard_locked( lock_path = self._storage_path.with_suffix(self._storage_path.suffix + ".lock") lock_handle = lock_path.open("a+", encoding="utf-8") fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) - disk_runs, disk_sequence, disk_evicted_runs, disk_evicted_runs_by_repo = self._read_from_disk_unlocked() + ( + disk_runs, + disk_sequence, + disk_evicted_runs, + disk_evicted_runs_by_repo, + disk_history_unknown, + ) = self._read_from_disk_unlocked() if guard_run_id and owner_repository: disk_entry = disk_runs.get(guard_run_id) disk_owner = _entry_owner_repository(disk_entry) if isinstance(disk_entry, dict) else "" if disk_owner and disk_owner != owner_repository: raise PermissionError("automation run_id belongs to another repository") + self._drop_runs_evicted_on_disk_locked(disk_runs, preserve_run_id=guard_run_id) for run_id, disk_entry in disk_runs.items(): current = self._runs.get(run_id) if current is not None: @@ -322,6 +349,7 @@ def _persist_with_owner_guard_locked( self._sequence = max(self._sequence, disk_sequence, len(self._runs)) self._evicted_runs_count = max(self._evicted_runs_count, disk_evicted_runs) self._merge_evicted_runs_by_repo_locked(disk_evicted_runs_by_repo) + self._history_completeness_unknown = self._history_completeness_unknown or disk_history_unknown self._evict_old_runs_locked() self._write_to_disk_locked() finally: @@ -339,6 +367,7 @@ def _write_to_disk_locked(self) -> None: "sequence": self._sequence, "evicted_runs": self._evicted_runs_count, "evicted_runs_by_repo": self._evicted_runs_by_repo, + "history_completeness_unknown": self._history_completeness_unknown, "runs": self._runs, } tmp = self._storage_path.with_suffix(self._storage_path.suffix + ".tmp") @@ -441,6 +470,7 @@ def record( previous_sequence = self._sequence previous_evicted_runs_count = self._evicted_runs_count previous_evicted_runs_by_repo = dict(self._evicted_runs_by_repo) + previous_history_completeness_unknown = self._history_completeness_unknown current = self._runs.get(run_id) if current: current_owner = _entry_owner_repository(current) @@ -494,6 +524,7 @@ def record( self._sequence = previous_sequence self._evicted_runs_count = previous_evicted_runs_count self._evicted_runs_by_repo = previous_evicted_runs_by_repo + self._history_completeness_unknown = previous_history_completeness_unknown raise return self._public_entry(self._runs[run_id]) @@ -515,6 +546,7 @@ def snapshot(self, *, limit: int | None = 100, include_events: bool = False) -> max_events_per_run = self._max_events_per_run evicted_runs_count = self._evicted_runs_count evicted_runs_by_repo = dict(self._evicted_runs_by_repo) + history_completeness_unknown = self._history_completeness_unknown task_states = Counter(str(run.get("task_state", "")).strip().lower() for run in retained_runs if run.get("task_state")) suggested_actions = Counter( @@ -546,6 +578,7 @@ def snapshot(self, *, limit: int | None = 100, include_events: bool = False) -> "events_included": include_events, "evicted_runs": evicted_runs_count, "evicted_runs_by_repo": evicted_runs_by_repo, + "history_completeness_unknown": history_completeness_unknown, "may_be_truncated": evicted_runs_count > 0, }, }, diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index ba627c5b..07f2a563 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -256,6 +256,32 @@ def test_control_snapshot_ignores_other_repo_ledger_eviction(self) -> None: self.assertEqual(control["action"], "continue") self.assertTrue(control["execution"]["failure_history_complete"]) + def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> None: + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type( + "Ledger", + (), + { + "snapshot": lambda self, limit=None: { + "runs": [], + "summary": {"retention": {"history_completeness_unknown": True}}, + } + }, + )() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={"default": {"max_consecutive_failures": 2}}), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") + + self.assertEqual(control["action"], "escalate") + self.assertFalse(control["execution"]["failure_history_complete"]) + def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index e57b53c7..db92cebc 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -205,6 +205,49 @@ def test_persisted_ledger_eviction_count_is_not_double_counted(self) -> None: self.assertTrue(snapshot["summary"]["retention"]["may_be_truncated"]) self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-2", "run-3"}) + def test_persist_merge_does_not_recount_disk_evicted_local_runs(self) -> None: + with TemporaryDirectory() as tmp: + path = Path(tmp) / "automation_runs.json" + ledger_a = AutomationRunLedger(max_runs=2, storage_path=path) + with patch("service.automation_run_ledger.time.time", side_effect=[1.0, 2.0, 3.0]): + ledger_a.record("run-1", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + ledger_a.record("run-2", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + ledger_b = AutomationRunLedger(max_runs=2, storage_path=path) + ledger_a.record("run-3", "queued", metadata={"source_repository": "QuantStrategyLab/RepoB"}) + with patch("service.automation_run_ledger.time.time", return_value=4.0): + ledger_b.record("run-4", "queued", metadata={"source_repository": "QuantStrategyLab/RepoB"}) + snapshot = AutomationRunLedger(max_runs=2, storage_path=path).snapshot(limit=None) + + self.assertEqual(snapshot["summary"]["retention"]["evicted_runs"], 2) + self.assertEqual(snapshot["summary"]["retention"]["evicted_runs_by_repo"], {"quantstrategylab/repoa": 2}) + self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-3", "run-4"}) + + def test_pre_migration_ledger_marks_history_completeness_unknown(self) -> None: + with TemporaryDirectory() as tmp: + path = Path(tmp) / "automation_runs.json" + path.write_text( + json.dumps( + { + "schema_version": "automation_run_ledger.v1", + "sequence": 1, + "runs": { + "run-1": { + "run_id": "run-1", + "task_state": "failed", + "updated_at": 1.0, + "metadata": {"source_repository": "QuantStrategyLab/RepoA"}, + } + }, + } + ), + encoding="utf-8", + ) + + ledger = AutomationRunLedger(max_runs=2, storage_path=path) + snapshot = ledger.snapshot(limit=None) + + self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) + def test_update_preserves_control_fields_when_omitted(self) -> None: self.ledger.record( "run-1", From 8a60650e4210aa6f3aaf097d00ec4b18f70029dd Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 06:24:37 +0800 Subject: [PATCH 31/72] fix: fail closed on unreadable automation ledger Co-Authored-By: Codex --- service/automation_decision.py | 7 ++++++- service/automation_run_ledger.py | 16 +++++++++++----- tests/test_ai_gateway_service_get_routes.py | 2 +- tests/test_automation_decision.py | 16 ++++++++++++++++ tests/test_automation_run_ledger.py | 16 ++++++++++++++++ 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/service/automation_decision.py b/service/automation_decision.py index fcab40b7..4b403b23 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -442,7 +442,12 @@ def decide_automation_execution( "failure_history_complete": failure_history_complete, "human_review_required": human_review_required, "auto_fix_allowed": action == EXECUTION_RUN and effective_mode == MODE_REVIEW_AND_FIX and not human_review_required, - "auto_merge_allowed": action == EXECUTION_RUN and effective_autonomy == AUTONOMY_AUTO_MERGE and not human_review_required, + "auto_merge_allowed": ( + action == EXECUTION_RUN + and effective_mode == MODE_REVIEW_AND_FIX + and effective_autonomy == AUTONOMY_AUTO_MERGE + and not human_review_required + ), "defer": defer, "reasons": reasons or ["execution allowed"], } diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index d68f56fb..388e8f82 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -225,7 +225,10 @@ def __init__( self._load_from_disk() def _load_from_disk(self) -> None: - if self._storage_path is None or not self._storage_path.exists(): + if self._storage_path is None: + return + if not self._storage_path.exists(): + self._history_completeness_unknown = True return runs, sequence, evicted_runs, evicted_runs_by_repo, history_unknown = self._read_from_disk_unlocked() with self._lock: @@ -238,14 +241,14 @@ def _load_from_disk(self) -> None: def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, dict[str, int], bool]: if self._storage_path is None or not self._storage_path.exists(): - return {}, 0, 0, {}, False + return {}, 0, 0, {}, self._storage_path is not None try: payload = json.loads(self._storage_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): - return {}, 0, 0, {}, False + return {}, 0, 0, {}, True runs = payload.get("runs") if isinstance(payload, dict) else None if not isinstance(runs, dict): - return {}, 0, 0, {}, False + return {}, 0, 0, {}, True clean_runs = {str(key): value for key, value in runs.items() if isinstance(value, dict)} sequence = _safe_int(payload.get("sequence"), len(clean_runs)) history_unknown = "evicted_runs" not in payload or "evicted_runs_by_repo" not in payload @@ -278,7 +281,10 @@ def _persist_locked(self) -> None: self._persist_with_owner_guard_locked() def _refresh_from_disk_locked(self) -> None: - if self._storage_path is None or not self._storage_path.exists(): + if self._storage_path is None: + return + if not self._storage_path.exists(): + self._history_completeness_unknown = True return lock_handle = None try: diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 847940f9..f75c6900 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -672,7 +672,7 @@ def test_automation_triage_reports_retryable_incident(self) -> None: self.assertTrue(triage["retry_allowed"]) self.assertEqual(triage["recommended_action"], "retry") self.assertEqual(triage["file_risk"], "low") - self.assertEqual(triage["control"]["execution"]["action"], "review_only") + self.assertEqual(triage["control"]["execution"]["action"], "human_review") self.assertIn("run_id=incident-123", triage["summary"]) finally: server.shutdown() diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 2874d412..450a475b 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -82,6 +82,22 @@ def test_auto_merge_requires_matching_repo_autonomy(self) -> None: self.assertEqual(result["effective_autonomy"], "auto_merge") self.assertTrue(result["auto_merge_allowed"]) + def test_auto_merge_is_disabled_when_effective_mode_is_review_only(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode="auto_merge", + control_action=CONTROL_CONTINUE, + service_health="degraded", + quota_status="ok", + org_health_status="ok", + policy={"default": {"max_autonomy": "auto_merge"}}, + ) + + self.assertEqual(result["action"], EXECUTION_RUN) + self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) + self.assertFalse(result["auto_fix_allowed"]) + self.assertFalse(result["auto_merge_allowed"]) + def test_degraded_health_forces_review_only(self) -> None: result = decide_automation_execution( repo="QuantStrategyLab/AIAuditBridge", diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index db92cebc..f624222c 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -248,6 +248,22 @@ def test_pre_migration_ledger_marks_history_completeness_unknown(self) -> None: self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) + def test_missing_persisted_ledger_marks_history_completeness_unknown(self) -> None: + with TemporaryDirectory() as tmp: + ledger = AutomationRunLedger(max_runs=2, storage_path=Path(tmp) / "missing.json") + snapshot = ledger.snapshot(limit=None) + + self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) + + def test_corrupt_persisted_ledger_marks_history_completeness_unknown(self) -> None: + with TemporaryDirectory() as tmp: + path = Path(tmp) / "automation_runs.json" + path.write_text("{not-json", encoding="utf-8") + ledger = AutomationRunLedger(max_runs=2, storage_path=path) + snapshot = ledger.snapshot(limit=None) + + self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) + def test_update_preserves_control_fields_when_omitted(self) -> None: self.ledger.record( "run-1", From 689b1a590623a332decf1c34d3196d75b2ddc5ac Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 06:35:12 +0800 Subject: [PATCH 32/72] fix: align automation triage fail-closed gates Co-Authored-By: Codex --- service/ai_gateway_service.py | 6 +++-- service/automation_run_ledger.py | 13 ++++++++--- tests/test_ai_gateway_automation_control.py | 25 ++++++++++++++++++++- tests/test_ai_gateway_service_get_routes.py | 2 +- tests/test_automation_run_ledger.py | 12 +++++++++- 5 files changed, 50 insertions(+), 8 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 4ad429e5..a85ed754 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -640,6 +640,8 @@ def _automation_triage_snapshot( category = service_failure_category(error) control_action = str(control.get("action") or CONTROL_REVIEW_ONLY) + execution = control.get("execution") if isinstance(control.get("execution"), dict) else {} + execution_auto_fix_allowed = bool(control.get("auto_fix_allowed")) and bool(execution.get("auto_fix_allowed")) retry_allowed = False deploy_allowed = False auto_fix_allowed = False @@ -671,7 +673,7 @@ def _automation_triage_snapshot( incident_class = "degraded" recommended_action = "open_issue" next_step = "pause_auto_fix" - elif control_action == CONTROL_CONTINUE: + elif control_action == CONTROL_CONTINUE and execution_auto_fix_allowed: incident_class = "investigate" recommended_action = "open_fix_pr" if path_risk in {RISK_LOW, RISK_MEDIUM} else "open_issue" next_step = "open_fix_pr" if path_risk in {RISK_LOW, RISK_MEDIUM} else "open_issue" @@ -680,7 +682,7 @@ def _automation_triage_snapshot( recommended_action = "open_issue" next_step = "open_issue" - if control_action == CONTROL_CONTINUE and category == "" and path_risk in {RISK_LOW, RISK_MEDIUM}: + if execution_auto_fix_allowed and category == "" and path_risk in {RISK_LOW, RISK_MEDIUM}: auto_fix_allowed = True deploy_allowed = True if path_risk in {RISK_HIGH, RISK_CRITICAL}: diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 388e8f82..4706a3ce 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -221,6 +221,7 @@ def __init__( self._evicted_runs_count = 0 self._evicted_runs_by_repo: dict[str, int] = {} self._history_completeness_unknown = False + self._storage_file_seen = False self._storage_path = storage_path self._load_from_disk() @@ -228,10 +229,10 @@ def _load_from_disk(self) -> None: if self._storage_path is None: return if not self._storage_path.exists(): - self._history_completeness_unknown = True return runs, sequence, evicted_runs, evicted_runs_by_repo, history_unknown = self._read_from_disk_unlocked() with self._lock: + self._storage_file_seen = True self._runs = runs self._sequence = sequence self._evicted_runs_count = evicted_runs @@ -241,7 +242,7 @@ def _load_from_disk(self) -> None: def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, dict[str, int], bool]: if self._storage_path is None or not self._storage_path.exists(): - return {}, 0, 0, {}, self._storage_path is not None + return {}, 0, 0, {}, self._storage_file_seen try: payload = json.loads(self._storage_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): @@ -284,7 +285,8 @@ def _refresh_from_disk_locked(self) -> None: if self._storage_path is None: return if not self._storage_path.exists(): - self._history_completeness_unknown = True + if self._storage_file_seen: + self._history_completeness_unknown = True return lock_handle = None try: @@ -299,6 +301,7 @@ def _refresh_from_disk_locked(self) -> None: disk_evicted_runs_by_repo, disk_history_unknown, ) = self._read_from_disk_unlocked() + self._storage_file_seen = True self._drop_runs_evicted_on_disk_locked(disk_runs) for run_id, disk_entry in disk_runs.items(): current = self._runs.get(run_id) @@ -340,6 +343,7 @@ def _persist_with_owner_guard_locked( disk_evicted_runs_by_repo, disk_history_unknown, ) = self._read_from_disk_unlocked() + self._storage_file_seen = True if guard_run_id and owner_repository: disk_entry = disk_runs.get(guard_run_id) disk_owner = _entry_owner_repository(disk_entry) if isinstance(disk_entry, dict) else "" @@ -383,6 +387,7 @@ def _write_to_disk_locked(self) -> None: except OSError: pass os.replace(tmp, self._storage_path) + self._storage_file_seen = True def _merge_entry_locked(self, current: dict[str, Any], disk_entry: dict[str, Any]) -> dict[str, Any]: current_terminal = _is_terminal_task_state(current.get("task_state")) @@ -477,6 +482,7 @@ def record( previous_evicted_runs_count = self._evicted_runs_count previous_evicted_runs_by_repo = dict(self._evicted_runs_by_repo) previous_history_completeness_unknown = self._history_completeness_unknown + previous_storage_file_seen = self._storage_file_seen current = self._runs.get(run_id) if current: current_owner = _entry_owner_repository(current) @@ -531,6 +537,7 @@ def record( self._evicted_runs_count = previous_evicted_runs_count self._evicted_runs_by_repo = previous_evicted_runs_by_repo self._history_completeness_unknown = previous_history_completeness_unknown + self._storage_file_seen = previous_storage_file_seen raise return self._public_entry(self._runs[run_id]) diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 07f2a563..ac601b18 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -9,7 +9,7 @@ import unittest from unittest.mock import patch -from service.ai_gateway_service import _automation_control_snapshot +from service.ai_gateway_service import _automation_control_snapshot, _automation_triage_snapshot class TestAutomationControlSnapshot(unittest.TestCase): @@ -305,6 +305,29 @@ def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(se self.assertTrue(control["execution"]["auto_fix_allowed"]) self.assertFalse(control["execution"]["auto_merge_allowed"]) + def test_triage_does_not_allow_auto_fix_when_execution_disallows_it(self) -> None: + control = { + "action": "continue", + "auto_fix_allowed": False, + "requires_human_review": False, + "service_health": "healthy", + "quota_status": "ok", + "org_health_status": "ok", + "execution": {"auto_fix_allowed": False}, + } + + with patch("service.ai_gateway_service._automation_control_snapshot", return_value=control): + triage = _automation_triage_snapshot( + "QuantStrategyLab/TargetRepo", + task="monthly", + changed_paths=["docs/runbook.md"], + ) + + self.assertFalse(triage["auto_fix_allowed"]) + self.assertFalse(triage["deploy_allowed"]) + self.assertEqual(triage["recommended_action"], "open_issue") + self.assertEqual(triage["next_step"], "open_issue") + def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: runs = [ { diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index f75c6900..847940f9 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -672,7 +672,7 @@ def test_automation_triage_reports_retryable_incident(self) -> None: self.assertTrue(triage["retry_allowed"]) self.assertEqual(triage["recommended_action"], "retry") self.assertEqual(triage["file_risk"], "low") - self.assertEqual(triage["control"]["execution"]["action"], "human_review") + self.assertEqual(triage["control"]["execution"]["action"], "review_only") self.assertIn("run_id=incident-123", triage["summary"]) finally: server.shutdown() diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index f624222c..2167d58f 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -248,11 +248,21 @@ def test_pre_migration_ledger_marks_history_completeness_unknown(self) -> None: self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) - def test_missing_persisted_ledger_marks_history_completeness_unknown(self) -> None: + def test_fresh_missing_persisted_ledger_starts_as_complete_empty_history(self) -> None: with TemporaryDirectory() as tmp: ledger = AutomationRunLedger(max_runs=2, storage_path=Path(tmp) / "missing.json") snapshot = ledger.snapshot(limit=None) + self.assertFalse(snapshot["summary"]["retention"]["history_completeness_unknown"]) + + def test_disappeared_persisted_ledger_marks_history_completeness_unknown(self) -> None: + with TemporaryDirectory() as tmp: + path = Path(tmp) / "automation_runs.json" + ledger = AutomationRunLedger(max_runs=2, storage_path=path) + ledger.record("run-1", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + path.unlink() + snapshot = ledger.snapshot(limit=None) + self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) def test_corrupt_persisted_ledger_marks_history_completeness_unknown(self) -> None: From b238d6d6dbefb0874b44aa44b0a5c51487700b6d Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 06:48:40 +0800 Subject: [PATCH 33/72] fix: keep automation run modes fail closed Co-Authored-By: Codex --- service/ai_gateway_service.py | 18 ++++++----- service/automation_decision.py | 2 ++ tests/test_ai_gateway_automation_control.py | 18 +++++++++++ tests/test_ai_gateway_service_get_routes.py | 33 +++++++++++++++++++-- tests/test_automation_decision.py | 8 +++++ 5 files changed, 70 insertions(+), 9 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index a85ed754..991d26b3 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -578,11 +578,7 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX - elif ( - execution.get("requested_mode") == MODE_REVIEW_AND_FIX - and execution.get("effective_mode") == MODE_REVIEW_ONLY - and strict_action == CONTROL_CONTINUE - ): + elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: strict_action = CONTROL_REVIEW_ONLY if strict_action != original_action: control["action"] = strict_action @@ -1629,8 +1625,15 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st _assert_automation_run_access(existing, claims) task_name = str(payload.get("task") or payload.get("task_name") or "") task_state = str(payload.get("task_state") or payload.get("state") or "running") - requested_mode = _normalize_control_mode_param(str(payload.get("mode") or MODE_REVIEW_AND_FIX)) - if not requested_mode: + existing_metadata = existing.get("metadata") if isinstance(existing, dict) and isinstance(existing.get("metadata"), dict) else {} + mode_from_payload = "mode" in payload + raw_mode = ( + payload.get("mode") + if mode_from_payload + else existing_metadata.get("requested_mode") or existing_metadata.get("mode") + ) + requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_ONLY)) + if mode_from_payload and not requested_mode: raise ValueError("invalid mode") run_metadata = { **metadata, @@ -1638,6 +1641,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st "repository": repo, "source_repository": source_repo, "caller_repository": str(claims.get("repository") or ""), + "requested_mode": requested_mode, } control = _automation_control_snapshot( repo, diff --git a/service/automation_decision.py b/service/automation_decision.py index 4b403b23..081d85fe 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -247,6 +247,8 @@ def load_execution_policy(path: Path | None = None) -> dict[str, Any]: if not configured: return _fail_closed_policy("execution policy path is not configured") path = Path(configured).expanduser() + if not path.is_absolute(): + return _fail_closed_policy("execution policy path must be absolute") if require_trusted_path: raw, read_error = _read_trusted_policy_file(path) if read_error: diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index ac601b18..228200e3 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -49,6 +49,24 @@ def test_control_snapshot_preserves_continue_for_explicit_review_and_fix_mode(se self.assertEqual(control["execution"]["effective_mode"], "review_and_fix") self.assertTrue(control["execution"]["auto_fix_allowed"]) + def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> None: + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={}), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_only") + + self.assertEqual(control["action"], "review_only") + self.assertEqual(control["execution"]["effective_mode"], "review_only") + self.assertFalse(control["auto_fix_allowed"]) + def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: with TemporaryDirectory() as tmp: policy_path = Path(tmp) / "execution_policy.json" diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 847940f9..ad6cc264 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -583,11 +583,40 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: recorded = json.loads(response.read().decode("utf-8")) self.assertEqual(recorded["run"]["run_id"], "platform-health-run-1") self.assertEqual(recorded["run"]["suggested_action"], recorded["control"]["action"]) - self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_and_fix") + self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_only") + self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_only") self.assertEqual(recorded["run"]["service_health"], recorded["control"]["service_health"]) self.assertEqual(recorded["run"]["quota_status"], recorded["control"]["quota_status"]) self.assertEqual(recorded["run"]["org_health_status"], recorded["control"]["org_health_status"]) + manual_payload = {**payload, "run_id": "platform-health-run-manual", "mode": "manual"} + manual_request = urllib.request.Request( + f"{base_url}/v1/ai/automation/runs", + data=json.dumps(manual_payload).encode("utf-8"), + method="POST", + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(manual_request, timeout=5) as response: + manual_recorded = json.loads(response.read().decode("utf-8")) + self.assertEqual(manual_recorded["control"]["execution"]["requested_mode"], "review_only") + self.assertEqual(manual_recorded["run"]["metadata"]["requested_mode"], "manual") + + manual_update_payload = { + **payload, + "run_id": "platform-health-run-manual", + "task_state": "failed", + } + manual_update_request = urllib.request.Request( + f"{base_url}/v1/ai/automation/runs", + data=json.dumps(manual_update_payload).encode("utf-8"), + method="POST", + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(manual_update_request, timeout=5) as response: + manual_updated = json.loads(response.read().decode("utf-8")) + self.assertEqual(manual_updated["control"]["execution"]["requested_mode"], "review_only") + self.assertEqual(manual_updated["run"]["metadata"]["requested_mode"], "manual") + invalid_mode_payload = {**payload, "run_id": "platform-health-run-invalid", "mode": "bad"} invalid_mode_request = urllib.request.Request( f"{base_url}/v1/ai/automation/runs", @@ -601,7 +630,7 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: with urllib.request.urlopen(f"{base_url}/v1/ai/automation/runs?include_events=true", timeout=5) as response: ledger = json.loads(response.read().decode("utf-8"))["ledger"] - self.assertEqual(ledger["summary"]["total_runs"], 1) + self.assertEqual(ledger["summary"]["total_runs"], 2) self.assertEqual(ledger["runs"][0]["task_name"], "platform-health") self.assertIn("events", ledger["runs"][0]) diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 450a475b..3647b92e 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -364,6 +364,14 @@ def test_load_execution_policy_fails_closed_when_path_unset(self) -> None: with patch.dict(os.environ, {}, clear=True): self.assertEqual(load_execution_policy()["default"]["max_autonomy"], "manual") + def test_load_execution_policy_fails_closed_for_relative_env_path(self) -> None: + env = { + "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH": "policy.json", + "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_OWNER": f"{os.getuid()}:{os.getgid()}", + } + with patch.dict(os.environ, env, clear=True): + self.assertEqual(load_execution_policy()["default"]["max_autonomy"], "manual") + def test_load_execution_policy_fails_closed_for_untrusted_env_path(self) -> None: with TemporaryDirectory() as tmp: path = Path(tmp) / "policy.json" From ef194a3caee0ff53774501b20d6b093d906b16ac Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 06:54:38 +0800 Subject: [PATCH 34/72] fix: recover ledger migration safety state Co-Authored-By: Codex --- service/ai_gateway_service.py | 5 ++-- service/automation_run_ledger.py | 15 ++++++++++- tests/test_ai_gateway_automation_control.py | 22 ++++++++++++++++ tests/test_automation_run_ledger.py | 29 +++++++++++++++++++++ 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 991d26b3..5d9d0564 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -615,7 +615,7 @@ def _automation_triage_snapshot( repo: str, *, task: str = "", - requested_mode: str = MODE_REVIEW_ONLY, + requested_mode: str = MODE_REVIEW_AND_FIX, failure_category: str = "", error: str = "", changed_paths: list[str] | None = None, @@ -1540,7 +1540,8 @@ def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, A params = parse_qs(parsed.query) repo = str(payload.get("source_repository") or params.get("repo", [""])[0] or claims.get("repository") or "unknown") task = str(payload.get("task") or params.get("task", [""])[0] or "") - requested_mode = _normalize_control_mode_param(str(payload.get("mode") or params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY)) + raw_mode = payload.get("mode") if "mode" in payload else params.get("mode", [MODE_REVIEW_AND_FIX])[0] + requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_AND_FIX)) if not requested_mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 4706a3ce..6a5d9f19 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -221,6 +221,7 @@ def __init__( self._evicted_runs_count = 0 self._evicted_runs_by_repo: dict[str, int] = {} self._history_completeness_unknown = False + self._history_completeness_unknown_from_legacy_schema = False self._storage_file_seen = False self._storage_path = storage_path self._load_from_disk() @@ -246,13 +247,19 @@ def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, try: payload = json.loads(self._storage_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): + self._history_completeness_unknown_from_legacy_schema = False return {}, 0, 0, {}, True runs = payload.get("runs") if isinstance(payload, dict) else None if not isinstance(runs, dict): + self._history_completeness_unknown_from_legacy_schema = False return {}, 0, 0, {}, True clean_runs = {str(key): value for key, value in runs.items() if isinstance(value, dict)} sequence = _safe_int(payload.get("sequence"), len(clean_runs)) - history_unknown = "evicted_runs" not in payload or "evicted_runs_by_repo" not in payload + legacy_retention_metadata_missing = "evicted_runs" not in payload or "evicted_runs_by_repo" not in payload + self._history_completeness_unknown_from_legacy_schema = ( + legacy_retention_metadata_missing and "history_completeness_unknown" not in payload + ) + history_unknown = legacy_retention_metadata_missing evicted_runs = _safe_int(payload.get("evicted_runs"), 0) raw_evicted_by_repo = payload.get("evicted_runs_by_repo") evicted_by_repo = { @@ -287,6 +294,7 @@ def _refresh_from_disk_locked(self) -> None: if not self._storage_path.exists(): if self._storage_file_seen: self._history_completeness_unknown = True + self._history_completeness_unknown_from_legacy_schema = False return lock_handle = None try: @@ -372,6 +380,9 @@ def _persist_with_owner_guard_locked( def _write_to_disk_locked(self) -> None: if self._storage_path is None: return + if self._history_completeness_unknown_from_legacy_schema: + self._history_completeness_unknown = False + self._history_completeness_unknown_from_legacy_schema = False payload = { "schema_version": "automation_run_ledger.v1", "sequence": self._sequence, @@ -482,6 +493,7 @@ def record( previous_evicted_runs_count = self._evicted_runs_count previous_evicted_runs_by_repo = dict(self._evicted_runs_by_repo) previous_history_completeness_unknown = self._history_completeness_unknown + previous_history_completeness_unknown_from_legacy_schema = self._history_completeness_unknown_from_legacy_schema previous_storage_file_seen = self._storage_file_seen current = self._runs.get(run_id) if current: @@ -537,6 +549,7 @@ def record( self._evicted_runs_count = previous_evicted_runs_count self._evicted_runs_by_repo = previous_evicted_runs_by_repo self._history_completeness_unknown = previous_history_completeness_unknown + self._history_completeness_unknown_from_legacy_schema = previous_history_completeness_unknown_from_legacy_schema self._storage_file_seen = previous_storage_file_seen raise return self._public_entry(self._runs[run_id]) diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 228200e3..53da07da 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -346,6 +346,28 @@ def test_triage_does_not_allow_auto_fix_when_execution_disallows_it(self) -> Non self.assertEqual(triage["recommended_action"], "open_issue") self.assertEqual(triage["next_step"], "open_issue") + def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() + ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={}), + ): + triage = _automation_triage_snapshot( + "QuantStrategyLab/TargetRepo", + task="monthly", + changed_paths=["docs/runbook.md"], + ) + + self.assertEqual(triage["control"]["execution"]["requested_mode"], "review_and_fix") + self.assertTrue(triage["auto_fix_allowed"]) + self.assertEqual(triage["recommended_action"], "open_fix_pr") + def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: runs = [ { diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index 2167d58f..967d4a3e 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -248,6 +248,35 @@ def test_pre_migration_ledger_marks_history_completeness_unknown(self) -> None: self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) + def test_pre_migration_ledger_recovers_after_schema_rewrite(self) -> None: + with TemporaryDirectory() as tmp: + path = Path(tmp) / "automation_runs.json" + path.write_text( + json.dumps( + { + "schema_version": "automation_run_ledger.v1", + "sequence": 1, + "runs": { + "run-1": { + "run_id": "run-1", + "task_state": "failed", + "updated_at": 1.0, + "metadata": {"source_repository": "QuantStrategyLab/RepoA"}, + } + }, + } + ), + encoding="utf-8", + ) + + ledger = AutomationRunLedger(max_runs=3, storage_path=path) + self.assertTrue(ledger.snapshot(limit=None)["summary"]["retention"]["history_completeness_unknown"]) + ledger.record("run-2", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + snapshot = AutomationRunLedger(max_runs=3, storage_path=path).snapshot(limit=None) + + self.assertFalse(snapshot["summary"]["retention"]["history_completeness_unknown"]) + self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-1", "run-2"}) + def test_fresh_missing_persisted_ledger_starts_as_complete_empty_history(self) -> None: with TemporaryDirectory() as tmp: ledger = AutomationRunLedger(max_runs=2, storage_path=Path(tmp) / "missing.json") From 6818446c2fe70b9dd67e39e675e3c14c0fc64384 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 07:00:32 +0800 Subject: [PATCH 35/72] fix: align execution action with review-only mode Co-Authored-By: Codex --- service/ai_gateway_service.py | 3 ++- service/automation_decision.py | 2 ++ tests/test_ai_gateway_service_get_routes.py | 4 ++-- tests/test_automation_decision.py | 4 +++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 5d9d0564..c8f44915 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -1633,7 +1633,8 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st if mode_from_payload else existing_metadata.get("requested_mode") or existing_metadata.get("mode") ) - requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_ONLY)) + default_mode = MODE_REVIEW_ONLY if existing is not None else MODE_REVIEW_AND_FIX + requested_mode = _normalize_control_mode_param(str(raw_mode or default_mode)) if mode_from_payload and not requested_mode: raise ValueError("invalid mode") run_metadata = { diff --git a/service/automation_decision.py b/service/automation_decision.py index 081d85fe..62480fb4 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -424,6 +424,8 @@ def decide_automation_execution( else: reasons.append(f"quota status is {quota}; execution already blocked") effective_mode = MODE_REVIEW_ONLY + if action == EXECUTION_RUN and effective_mode == MODE_REVIEW_ONLY: + action = EXECUTION_REVIEW_ONLY human_review_required = action == EXECUTION_HUMAN_REVIEW return { diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index ad6cc264..70b0e4c4 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -583,8 +583,8 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: recorded = json.loads(response.read().decode("utf-8")) self.assertEqual(recorded["run"]["run_id"], "platform-health-run-1") self.assertEqual(recorded["run"]["suggested_action"], recorded["control"]["action"]) - self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_only") - self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_only") + self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_and_fix") + self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_and_fix") self.assertEqual(recorded["run"]["service_health"], recorded["control"]["service_health"]) self.assertEqual(recorded["run"]["quota_status"], recorded["control"]["quota_status"]) self.assertEqual(recorded["run"]["org_health_status"], recorded["control"]["org_health_status"]) diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 3647b92e..9a11a2b8 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -93,7 +93,7 @@ def test_auto_merge_is_disabled_when_effective_mode_is_review_only(self) -> None policy={"default": {"max_autonomy": "auto_merge"}}, ) - self.assertEqual(result["action"], EXECUTION_RUN) + self.assertEqual(result["action"], EXECUTION_REVIEW_ONLY) self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) self.assertFalse(result["auto_fix_allowed"]) self.assertFalse(result["auto_merge_allowed"]) @@ -108,6 +108,7 @@ def test_degraded_health_forces_review_only(self) -> None: org_health_status="ok", ) + self.assertEqual(result["action"], EXECUTION_REVIEW_ONLY) self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) self.assertFalse(result["auto_fix_allowed"]) self.assertFalse(result["human_review_required"]) @@ -197,6 +198,7 @@ def test_repo_policy_can_force_review_only(self) -> None: policy={"repositories": {"QuantStrategyLab/CryptoLivePoolPipelines": {"max_autonomy": "review_only"}}}, ) + self.assertEqual(result["action"], EXECUTION_REVIEW_ONLY) self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) self.assertFalse(result["human_review_required"]) self.assertFalse(result["auto_fix_allowed"]) From 5f0ef5511d19fd8ec106b8eeed5e764dea6e6bf5 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 07:07:45 +0800 Subject: [PATCH 36/72] fix: prevent stale ledger run resurrection Co-Authored-By: Codex --- service/automation_run_ledger.py | 28 +++++++++++----------------- tests/test_automation_run_ledger.py | 21 +++++++++++++++++++-- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 6a5d9f19..2cd32b61 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -221,7 +221,6 @@ def __init__( self._evicted_runs_count = 0 self._evicted_runs_by_repo: dict[str, int] = {} self._history_completeness_unknown = False - self._history_completeness_unknown_from_legacy_schema = False self._storage_file_seen = False self._storage_path = storage_path self._load_from_disk() @@ -247,19 +246,13 @@ def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, try: payload = json.loads(self._storage_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): - self._history_completeness_unknown_from_legacy_schema = False return {}, 0, 0, {}, True runs = payload.get("runs") if isinstance(payload, dict) else None if not isinstance(runs, dict): - self._history_completeness_unknown_from_legacy_schema = False return {}, 0, 0, {}, True clean_runs = {str(key): value for key, value in runs.items() if isinstance(value, dict)} sequence = _safe_int(payload.get("sequence"), len(clean_runs)) - legacy_retention_metadata_missing = "evicted_runs" not in payload or "evicted_runs_by_repo" not in payload - self._history_completeness_unknown_from_legacy_schema = ( - legacy_retention_metadata_missing and "history_completeness_unknown" not in payload - ) - history_unknown = legacy_retention_metadata_missing + history_unknown = "evicted_runs" not in payload or "evicted_runs_by_repo" not in payload evicted_runs = _safe_int(payload.get("evicted_runs"), 0) raw_evicted_by_repo = payload.get("evicted_runs_by_repo") evicted_by_repo = { @@ -294,7 +287,6 @@ def _refresh_from_disk_locked(self) -> None: if not self._storage_path.exists(): if self._storage_file_seen: self._history_completeness_unknown = True - self._history_completeness_unknown_from_legacy_schema = False return lock_handle = None try: @@ -334,6 +326,7 @@ def _persist_with_owner_guard_locked( *, guard_run_id: str = "", owner_repository: str = "", + guard_preexisting: bool = False, ) -> None: if self._storage_path is None: return @@ -352,11 +345,13 @@ def _persist_with_owner_guard_locked( disk_history_unknown, ) = self._read_from_disk_unlocked() self._storage_file_seen = True - if guard_run_id and owner_repository: + if guard_run_id: disk_entry = disk_runs.get(guard_run_id) disk_owner = _entry_owner_repository(disk_entry) if isinstance(disk_entry, dict) else "" - if disk_owner and disk_owner != owner_repository: + if owner_repository and disk_owner and disk_owner != owner_repository: raise PermissionError("automation run_id belongs to another repository") + if guard_preexisting and disk_entry is None and len(disk_runs) >= self._max_runs: + raise ValueError("automation run was evicted from retained ledger") self._drop_runs_evicted_on_disk_locked(disk_runs, preserve_run_id=guard_run_id) for run_id, disk_entry in disk_runs.items(): current = self._runs.get(run_id) @@ -380,9 +375,6 @@ def _persist_with_owner_guard_locked( def _write_to_disk_locked(self) -> None: if self._storage_path is None: return - if self._history_completeness_unknown_from_legacy_schema: - self._history_completeness_unknown = False - self._history_completeness_unknown_from_legacy_schema = False payload = { "schema_version": "automation_run_ledger.v1", "sequence": self._sequence, @@ -493,7 +485,6 @@ def record( previous_evicted_runs_count = self._evicted_runs_count previous_evicted_runs_by_repo = dict(self._evicted_runs_by_repo) previous_history_completeness_unknown = self._history_completeness_unknown - previous_history_completeness_unknown_from_legacy_schema = self._history_completeness_unknown_from_legacy_schema previous_storage_file_seen = self._storage_file_seen current = self._runs.get(run_id) if current: @@ -542,14 +533,17 @@ def record( if self._storage_path is None: self._evict_old_runs_locked() try: - self._persist_with_owner_guard_locked(guard_run_id=run_id, owner_repository=owner_repository) + self._persist_with_owner_guard_locked( + guard_run_id=run_id, + owner_repository=owner_repository, + guard_preexisting=current is not None, + ) except Exception: self._runs = previous_runs self._sequence = previous_sequence self._evicted_runs_count = previous_evicted_runs_count self._evicted_runs_by_repo = previous_evicted_runs_by_repo self._history_completeness_unknown = previous_history_completeness_unknown - self._history_completeness_unknown_from_legacy_schema = previous_history_completeness_unknown_from_legacy_schema self._storage_file_seen = previous_storage_file_seen raise return self._public_entry(self._runs[run_id]) diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index 967d4a3e..7395b55c 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -248,7 +248,7 @@ def test_pre_migration_ledger_marks_history_completeness_unknown(self) -> None: self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) - def test_pre_migration_ledger_recovers_after_schema_rewrite(self) -> None: + def test_pre_migration_ledger_keeps_history_unknown_after_schema_rewrite(self) -> None: with TemporaryDirectory() as tmp: path = Path(tmp) / "automation_runs.json" path.write_text( @@ -274,9 +274,26 @@ def test_pre_migration_ledger_recovers_after_schema_rewrite(self) -> None: ledger.record("run-2", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) snapshot = AutomationRunLedger(max_runs=3, storage_path=path).snapshot(limit=None) - self.assertFalse(snapshot["summary"]["retention"]["history_completeness_unknown"]) + self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-1", "run-2"}) + def test_stale_update_cannot_resurrect_run_evicted_on_disk(self) -> None: + with TemporaryDirectory() as tmp: + path = Path(tmp) / "automation_runs.json" + ledger_a = AutomationRunLedger(max_runs=2, storage_path=path) + with patch("service.automation_run_ledger.time.time", side_effect=[1.0, 2.0]): + ledger_a.record("run-1", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + ledger_a.record("run-2", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + ledger_b = AutomationRunLedger(max_runs=2, storage_path=path) + with patch("service.automation_run_ledger.time.time", return_value=3.0): + ledger_a.record("run-3", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + with patch("service.automation_run_ledger.time.time", return_value=4.0): + with self.assertRaises(ValueError): + ledger_b.record("run-1", "failed", metadata={"source_repository": "QuantStrategyLab/RepoA"}) + snapshot = AutomationRunLedger(max_runs=2, storage_path=path).snapshot(limit=None) + + self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-2", "run-3"}) + def test_fresh_missing_persisted_ledger_starts_as_complete_empty_history(self) -> None: with TemporaryDirectory() as tmp: ledger = AutomationRunLedger(max_runs=2, storage_path=Path(tmp) / "missing.json") From ef1f41e64bcf905d1bbb23239991b22c1857f53d Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 07:09:36 +0800 Subject: [PATCH 37/72] docs: compress autonomy deployment notes Co-Authored-By: Codex --- docs/ai_autonomy_architecture.md | 9 +-------- docs/async_service_deployment.md | 27 +-------------------------- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/docs/ai_autonomy_architecture.md b/docs/ai_autonomy_architecture.md index f786c986..e2f91131 100644 --- a/docs/ai_autonomy_architecture.md +++ b/docs/ai_autonomy_architecture.md @@ -337,14 +337,7 @@ AIAuditBridge 是 QuantStrategyLab 的 AI 审计控制面，负责： - 健康驱动的执行降级策略； - repo 级别自治阈值。 -首批落地边界： - -- `/v1/ai/automation/control` 输出 `execution` 决策快照； -- health degraded / runtime pause 时，执行模式降级到 `review_only`； -- quota low 时给出低成本模型建议，quota exhausted / blocked 时建议 defer； -- service-owned run 的 repo-wide 连续失败达到 repo 阈值时强制 human review； -- repo 级 `max_autonomy` / `max_consecutive_failures` 从服务端受控 policy 文件读取，不信任被审仓库 checkout； -- 该阶段只影响调度建议和控制面输出，不自动放宽 merge / deploy 权限。 +首批落地边界：`/v1/ai/automation/control` 输出 `execution` 决策快照；health/quota/failure streak 会降级到 `review_only`、低成本模型、defer 或 human review；repo 级 `max_autonomy` / `max_consecutive_failures` 只从服务端受控 policy 读取，不信任被审仓库 checkout，也不自动放宽 merge / deploy 权限。 ### Phase 4：扩大自动修复，但只扩大低风险面 diff --git a/docs/async_service_deployment.md b/docs/async_service_deployment.md index 1914dbe5..ec8b16a4 100644 --- a/docs/async_service_deployment.md +++ b/docs/async_service_deployment.md @@ -71,32 +71,7 @@ bash scripts/deploy_codex_audit_service.sh deploy ``` The job directory should be owned by the service user and mode `0700`. -The deploy script points `CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH` to -`/etc/codex-audit-bridge-policy/execution_policy.json` by default and creates -a conservative default file if it is missing. This admin-owned, service-readable -policy file sits outside the service-user-writable job workspace and can cap repo -autonomy without trusting the reviewed checkout: - -```json -{ - "default": { - "max_autonomy": "auto_pr", - "max_consecutive_failures": 3, - "low_cost_model": "gpt-5.4-mini", - "low_cost_provider": "openai" - }, - "repositories": { - "QuantStrategyLab/CryptoLivePoolPipelines": { - "max_autonomy": "review_only", - "max_consecutive_failures": 2 - } - } -} -``` - -If this configured file is missing, unreadable, malformed, or the policy path is -not configured, the service fails closed for execution decisions until the -configuration is repaired. +The deploy script points `CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH` to `/etc/codex-audit-bridge-policy/execution_policy.json` by default and creates a conservative admin-owned policy file if missing. The file lives outside the service-user-writable job workspace and can cap repo autonomy with `default` / `repositories` keys such as `max_autonomy`, `max_consecutive_failures`, `low_cost_model`, and `low_cost_provider`. If the configured file is missing, unreadable, malformed, or the policy path is unset, execution decisions fail closed until configuration is repaired. The service should rely on an authenticated Codex CLI session and must not inject OpenAI/Codex API keys into the Codex subprocess. From 251e3a02dc8f621e10b743e57a764585397005f7 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 07:19:23 +0800 Subject: [PATCH 38/72] fix: preserve legacy control action contract Co-Authored-By: Codex --- service/ai_gateway_service.py | 10 +++++----- service/automation_decision.py | 2 +- tests/test_ai_gateway_automation_control.py | 21 ++++++++++++++------- tests/test_ai_gateway_service_get_routes.py | 2 +- tests/test_automation_decision.py | 4 ++-- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index c8f44915..0137e434 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -580,10 +580,10 @@ def _automation_control_snapshot( strict_action = CONTROL_PAUSE_AUTO_FIX elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: strict_action = CONTROL_REVIEW_ONLY + control["effective_action"] = strict_action if strict_action != original_action: - control["action"] = strict_action reasons = control.get("reasons") if isinstance(control.get("reasons"), list) else [] - reasons.append("capped by execution decision") + reasons.append("effective action capped by execution decision") control["reasons"] = reasons control["auto_fix_allowed"] = bool(execution.get("auto_fix_allowed")) and strict_action == CONTROL_CONTINUE control["auto_merge_allowed"] = bool(execution.get("auto_merge_allowed")) and strict_action == CONTROL_CONTINUE @@ -635,7 +635,7 @@ def _automation_triage_snapshot( if not category and error: category = service_failure_category(error) - control_action = str(control.get("action") or CONTROL_REVIEW_ONLY) + control_action = str(control.get("effective_action") or control.get("action") or CONTROL_REVIEW_ONLY) execution = control.get("execution") if isinstance(control.get("execution"), dict) else {} execution_auto_fix_allowed = bool(control.get("auto_fix_allowed")) and bool(execution.get("auto_fix_allowed")) retry_allowed = False @@ -757,7 +757,7 @@ def _record_job_automation_run(job: dict[str, Any]) -> None: str(job.get("job_id") or ""), task_state, task_name=task_name, - suggested_action=str(control.get("action") or ""), + suggested_action=str(control.get("effective_action") or control.get("action") or ""), service_health=str(control.get("service_health") or ""), quota_status=str(control.get("quota_status") or ""), org_health_status=str(control.get("org_health_status") or ""), @@ -1660,7 +1660,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st run_id, task_state, task_name=task_name, - suggested_action=str(control.get("action") or ""), + suggested_action=str(control.get("effective_action") or control.get("action") or ""), service_health=str(control.get("service_health") or ""), quota_status=control.get("quota_status") or "", org_health_status=str(control.get("org_health_status") or ""), diff --git a/service/automation_decision.py b/service/automation_decision.py index 62480fb4..28c95eaf 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -32,7 +32,7 @@ EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH" EXECUTION_POLICY_OWNER_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_OWNER" POLICY_LOAD_ERROR_KEY = "_load_error" -TRUSTED_FAILURE_ORIGINS = frozenset({"service_job"}) +TRUSTED_FAILURE_ORIGINS = frozenset({"service_job", "external_workflow"}) POLICY_ALLOWED_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider", "quota_low_behavior"}) POLICY_REQUIRED_DEFAULT_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider"}) POLICY_LOW_QUOTA_BEHAVIORS = frozenset({"low_cost_model", "defer"}) diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 53da07da..e7410911 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -63,7 +63,8 @@ def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_only") - self.assertEqual(control["action"], "review_only") + self.assertEqual(control["action"], "continue") + self.assertEqual(control["effective_action"], "review_only") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["auto_fix_allowed"]) @@ -109,7 +110,8 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") - self.assertEqual(control["action"], "review_only") + self.assertEqual(control["action"], "continue") + self.assertEqual(control["effective_action"], "review_only") self.assertFalse(control["requires_human_review"]) self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) @@ -159,7 +161,8 @@ def snapshot(self, limit=100): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") self.assertIsNone(ledger.requested_limit) - self.assertEqual(control["action"], "escalate") + self.assertEqual(control["action"], "continue") + self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) @@ -196,7 +199,8 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None pending_run=pending_run, ) - self.assertEqual(control["action"], "escalate") + self.assertEqual(control["action"], "continue") + self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) @@ -237,7 +241,8 @@ def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "escalate") + self.assertEqual(control["action"], "continue") + self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertFalse(control["execution"]["failure_history_complete"]) self.assertEqual(control["execution"]["consecutive_failures"], 1) @@ -297,7 +302,8 @@ def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> N ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "escalate") + self.assertEqual(control["action"], "continue") + self.assertEqual(control["effective_action"], "escalate") self.assertFalse(control["execution"]["failure_history_complete"]) def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(self) -> None: @@ -439,7 +445,8 @@ def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") - self.assertEqual(control["action"], "escalate") + self.assertEqual(control["action"], "continue") + self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertFalse(control["execution"]["auto_merge_allowed"]) diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 70b0e4c4..80aae31c 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -582,7 +582,7 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: self.assertEqual(response.status, 200) recorded = json.loads(response.read().decode("utf-8")) self.assertEqual(recorded["run"]["run_id"], "platform-health-run-1") - self.assertEqual(recorded["run"]["suggested_action"], recorded["control"]["action"]) + self.assertEqual(recorded["run"]["suggested_action"], recorded["control"]["effective_action"]) self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_and_fix") self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_and_fix") self.assertEqual(recorded["run"]["service_health"], recorded["control"]["service_health"]) diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 9a11a2b8..556363a7 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -214,7 +214,7 @@ def test_failure_streak_matching_is_case_insensitive(self) -> None: self.assertEqual(consecutive_failure_count(runs, repo="quantstrategylab/aiauditbridge"), 1) - def test_external_workflow_failures_do_not_force_repo_failure_streak(self) -> None: + def test_external_workflow_failures_count_toward_repo_failure_streak(self) -> None: runs = [ { "task_name": "monthly", @@ -223,7 +223,7 @@ def test_external_workflow_failures_do_not_force_repo_failure_streak(self) -> No }, ] - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 0) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) def test_invalid_repo_autonomy_fails_closed(self) -> None: result = decide_automation_execution( From 8133b00741886f42b404c96de395f47c168b4b23 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 07:25:44 +0800 Subject: [PATCH 39/72] fix: enforce strict automation control action Co-Authored-By: Codex --- docs/ai_autonomy_architecture.md | 1 - docs/async_service_deployment.md | 1 - service/ai_gateway_service.py | 7 +++++-- tests/test_ai_gateway_automation_control.py | 14 +++++++------- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/ai_autonomy_architecture.md b/docs/ai_autonomy_architecture.md index e2f91131..35e47537 100644 --- a/docs/ai_autonomy_architecture.md +++ b/docs/ai_autonomy_architecture.md @@ -337,7 +337,6 @@ AIAuditBridge 是 QuantStrategyLab 的 AI 审计控制面，负责： - 健康驱动的执行降级策略； - repo 级别自治阈值。 -首批落地边界：`/v1/ai/automation/control` 输出 `execution` 决策快照；health/quota/failure streak 会降级到 `review_only`、低成本模型、defer 或 human review；repo 级 `max_autonomy` / `max_consecutive_failures` 只从服务端受控 policy 读取，不信任被审仓库 checkout，也不自动放宽 merge / deploy 权限。 ### Phase 4：扩大自动修复，但只扩大低风险面 diff --git a/docs/async_service_deployment.md b/docs/async_service_deployment.md index ec8b16a4..646fddeb 100644 --- a/docs/async_service_deployment.md +++ b/docs/async_service_deployment.md @@ -71,7 +71,6 @@ bash scripts/deploy_codex_audit_service.sh deploy ``` The job directory should be owned by the service user and mode `0700`. -The deploy script points `CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH` to `/etc/codex-audit-bridge-policy/execution_policy.json` by default and creates a conservative admin-owned policy file if missing. The file lives outside the service-user-writable job workspace and can cap repo autonomy with `default` / `repositories` keys such as `max_autonomy`, `max_consecutive_failures`, `low_cost_model`, and `low_cost_provider`. If the configured file is missing, unreadable, malformed, or the policy path is unset, execution decisions fail closed until configuration is repaired. The service should rely on an authenticated Codex CLI session and must not inject OpenAI/Codex API keys into the Codex subprocess. diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 0137e434..f90e80ef 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -580,10 +580,12 @@ def _automation_control_snapshot( strict_action = CONTROL_PAUSE_AUTO_FIX elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: strict_action = CONTROL_REVIEW_ONLY + control["runtime_action"] = original_action control["effective_action"] = strict_action if strict_action != original_action: + control["action"] = strict_action reasons = control.get("reasons") if isinstance(control.get("reasons"), list) else [] - reasons.append("effective action capped by execution decision") + reasons.append("capped by execution decision") control["reasons"] = reasons control["auto_fix_allowed"] = bool(execution.get("auto_fix_allowed")) and strict_action == CONTROL_CONTINUE control["auto_merge_allowed"] = bool(execution.get("auto_merge_allowed")) and strict_action == CONTROL_CONTINUE @@ -1625,7 +1627,8 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st raise PermissionError("automation run is service-owned") _assert_automation_run_access(existing, claims) task_name = str(payload.get("task") or payload.get("task_name") or "") - task_state = str(payload.get("task_state") or payload.get("state") or "running") + existing_state = str(existing.get("task_state") or "") if isinstance(existing, dict) else "" + task_state = str(payload.get("task_state") or payload.get("state") or existing_state or "running") existing_metadata = existing.get("metadata") if isinstance(existing, dict) and isinstance(existing.get("metadata"), dict) else {} mode_from_payload = "mode" in payload raw_mode = ( diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index e7410911..36960194 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -63,7 +63,7 @@ def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_only") - self.assertEqual(control["action"], "continue") + self.assertEqual(control["action"], "review_only") self.assertEqual(control["effective_action"], "review_only") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["auto_fix_allowed"]) @@ -110,7 +110,7 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") - self.assertEqual(control["action"], "continue") + self.assertEqual(control["action"], "review_only") self.assertEqual(control["effective_action"], "review_only") self.assertFalse(control["requires_human_review"]) self.assertEqual(control["execution"]["effective_mode"], "review_only") @@ -161,7 +161,7 @@ def snapshot(self, limit=100): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") self.assertIsNone(ledger.requested_limit) - self.assertEqual(control["action"], "continue") + self.assertEqual(control["action"], "escalate") self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) @@ -199,7 +199,7 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None pending_run=pending_run, ) - self.assertEqual(control["action"], "continue") + self.assertEqual(control["action"], "escalate") self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) @@ -241,7 +241,7 @@ def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "continue") + self.assertEqual(control["action"], "escalate") self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertFalse(control["execution"]["failure_history_complete"]) @@ -302,7 +302,7 @@ def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> N ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "continue") + self.assertEqual(control["action"], "escalate") self.assertEqual(control["effective_action"], "escalate") self.assertFalse(control["execution"]["failure_history_complete"]) @@ -445,7 +445,7 @@ def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") - self.assertEqual(control["action"], "continue") + self.assertEqual(control["action"], "escalate") self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertFalse(control["execution"]["auto_merge_allowed"]) From b12e407450c3bed8b1f7a8206bc15c2976ea52de Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 07:30:28 +0800 Subject: [PATCH 40/72] fix: preserve pause semantics in automation control Co-Authored-By: Codex --- service/ai_gateway_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index f90e80ef..d9c84fb0 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -574,7 +574,7 @@ def _automation_control_snapshot( strict_action = original_action if execution.get("action") == EXECUTION_HUMAN_REVIEW: strict_action = CONTROL_ESCALATE - elif execution.get("action") == EXECUTION_REVIEW_ONLY: + elif execution.get("action") == EXECUTION_REVIEW_ONLY and strict_action != CONTROL_PAUSE_AUTO_FIX: strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX @@ -1636,7 +1636,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st if mode_from_payload else existing_metadata.get("requested_mode") or existing_metadata.get("mode") ) - default_mode = MODE_REVIEW_ONLY if existing is not None else MODE_REVIEW_AND_FIX + default_mode = MODE_REVIEW_AND_FIX requested_mode = _normalize_control_mode_param(str(raw_mode or default_mode)) if mode_from_payload and not requested_mode: raise ValueError("invalid mode") From 9eaa83e9cb53f65e830880f6b221b942611bba9b Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 07:40:25 +0800 Subject: [PATCH 41/72] fix: bound automation history and policy trust checks Co-Authored-By: Codex --- docs/ai_autonomy_architecture.md | 1 - docs/async_service_deployment.md | 1 - service/ai_gateway_service.py | 12 +++++++++++- service/automation_decision.py | 18 +++++++++++------- tests/test_ai_gateway_automation_control.py | 20 +------------------- tests/test_ai_gateway_service_get_routes.py | 4 ++-- 6 files changed, 25 insertions(+), 31 deletions(-) diff --git a/docs/ai_autonomy_architecture.md b/docs/ai_autonomy_architecture.md index 35e47537..41902644 100644 --- a/docs/ai_autonomy_architecture.md +++ b/docs/ai_autonomy_architecture.md @@ -337,7 +337,6 @@ AIAuditBridge 是 QuantStrategyLab 的 AI 审计控制面，负责： - 健康驱动的执行降级策略； - repo 级别自治阈值。 - ### Phase 4：扩大自动修复，但只扩大低风险面 目标：提升无人值守覆盖率，但不放松安全门。 diff --git a/docs/async_service_deployment.md b/docs/async_service_deployment.md index 646fddeb..4a1a6b53 100644 --- a/docs/async_service_deployment.md +++ b/docs/async_service_deployment.md @@ -71,7 +71,6 @@ bash scripts/deploy_codex_audit_service.sh deploy ``` The job directory should be owned by the service user and mode `0700`. - The service should rely on an authenticated Codex CLI session and must not inject OpenAI/Codex API keys into the Codex subprocess. With `CODEX_AUDIT_SERVICE_CODEX_ACCOUNT_USAGE=1`, `/v1/ai/quota` includes a diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index d9c84fb0..af6cad37 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -537,7 +537,17 @@ def _automation_control_snapshot( repo_evictions = int(evicted_by_repo.get(str(repo or "unknown").strip().lower(), 0) or 0) except (TypeError, ValueError): repo_evictions = 0 - failure_history_complete = repo_evictions <= 0 and not bool(retention.get("history_completeness_unknown")) + normalized_repo = str(repo or "unknown").strip().lower() + repo_history_has_terminal_boundary = any( + str(run.get("task_state") or "").strip().lower() not in {"failed", "queued", "running", "pending", "in_progress"} + and _automation_run_owner_repository(run).strip().lower() == normalized_repo + for run in recent_runs + if isinstance(run, dict) + ) + failure_history_complete = ( + not bool(retention.get("history_completeness_unknown")) + and (repo_evictions <= 0 or repo_history_has_terminal_boundary) + ) ledger_unavailable = False except Exception: recent_runs = [] diff --git a/service/automation_decision.py b/service/automation_decision.py index 28c95eaf..73369fd5 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -141,10 +141,7 @@ def _policy_metadata_trust_error(info: os.stat_result, *, kind: str) -> str: def _policy_parent_trust_error(path: Path) -> str: expected_uid, expected_gid = _expected_policy_owner() - parents = [path.parent] - if expected_uid == 0 and expected_gid == 0: - parents.extend(path.parent.parents) - for parent in parents: + for index, parent in enumerate([path.parent, *path.parent.parents]): try: info = parent.lstat() except FileNotFoundError: @@ -155,9 +152,16 @@ def _policy_parent_trust_error(path: Path) -> str: return "execution policy parent directory is a symlink" if not stat.S_ISDIR(info.st_mode): return "execution policy parent path is not a directory" - trust_error = _policy_metadata_trust_error(info, kind="parent directory") - if trust_error: - return trust_error + if index == 0: + trust_error = _policy_metadata_trust_error(info, kind="parent directory") + if trust_error: + return trust_error + continue + owner_ok = (info.st_uid, info.st_gid) == (expected_uid, expected_gid) or info.st_uid == 0 + if not owner_ok: + return "execution policy parent directory owner is invalid" + if info.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + return "execution policy parent directory permissions are too broad" return "" diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 36960194..df6a803d 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -31,24 +31,6 @@ def test_control_snapshot_defaults_to_review_and_fix_for_healthy_repo(self) -> N self.assertEqual(control["execution"]["effective_mode"], "review_and_fix") self.assertTrue(control["execution"]["auto_fix_allowed"]) - def test_control_snapshot_preserves_continue_for_explicit_review_and_fix_mode(self) -> None: - health = type("Health", (), {"status": "healthy"})() - quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() - ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() - - with ( - patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), - patch("service.ai_gateway_service.get_health_monitor", return_value=health), - patch("service.ai_gateway_service.get_quota_manager", return_value=quota), - patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), - patch("service.ai_gateway_service.load_execution_policy", return_value={}), - ): - control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") - - self.assertEqual(control["action"], "continue") - self.assertEqual(control["execution"]["effective_mode"], "review_and_fix") - self.assertTrue(control["execution"]["auto_fix_allowed"]) - def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() @@ -69,7 +51,7 @@ def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> self.assertFalse(control["auto_fix_allowed"]) def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: - with TemporaryDirectory() as tmp: + with TemporaryDirectory(dir=".") as tmp: policy_path = Path(tmp) / "execution_policy.json" policy_path.write_text( json.dumps( diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 80aae31c..a6104a2a 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -184,7 +184,7 @@ def test_static_token_dashboard_can_read_allowlisted_repository_runs(self) -> No self.assertEqual([run["run_id"] for run in filtered["runs"]], ["run-a"]) def test_automation_run_update_rejects_owner_mismatch_even_for_operator(self) -> None: - with tempfile.TemporaryDirectory() as tmp: + with tempfile.TemporaryDirectory(dir=".") as tmp: env = { "CODEX_AUDIT_SERVICE_AUTH": "none", "CODEX_AUDIT_SERVICE_ALLOW_NO_AUTH_FOR_LOCAL_TESTS": "true", @@ -647,7 +647,7 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: server.server_close() def test_automation_triage_reports_retryable_incident(self) -> None: - with tempfile.TemporaryDirectory() as tmp: + with tempfile.TemporaryDirectory(dir=".") as tmp: policy_path = os.path.join(tmp, "execution_policy.json") with open(policy_path, "w", encoding="utf-8") as handle: json.dump( From efe2791f0fb49066518d11c260f5e450ad673a53 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 07:52:30 +0800 Subject: [PATCH 42/72] fix: align automation control defaults Co-Authored-By: Codex --- service/ai_gateway_service.py | 14 ++++---- tests/test_ai_gateway_automation_control.py | 39 +++++++++++---------- tests/test_ai_gateway_service_get_routes.py | 6 ++-- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index af6cad37..20d5c256 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -513,7 +513,7 @@ def _automation_control_snapshot( repo: str, *, task_name: str = "", - requested_mode: str = MODE_REVIEW_AND_FIX, + requested_mode: str = MODE_REVIEW_ONLY, pending_run: dict[str, Any] | None = None, ) -> dict[str, Any]: try: @@ -583,7 +583,7 @@ def _automation_control_snapshot( original_action = str(control.get("action") or CONTROL_REVIEW_ONLY) strict_action = original_action if execution.get("action") == EXECUTION_HUMAN_REVIEW: - strict_action = CONTROL_ESCALATE + strict_action = CONTROL_ESCALATE if original_action == CONTROL_ESCALATE or ledger_unavailable else CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_REVIEW_ONLY and strict_action != CONTROL_PAUSE_AUTO_FIX: strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: @@ -627,7 +627,7 @@ def _automation_triage_snapshot( repo: str, *, task: str = "", - requested_mode: str = MODE_REVIEW_AND_FIX, + requested_mode: str = MODE_REVIEW_ONLY, failure_category: str = "", error: str = "", changed_paths: list[str] | None = None, @@ -1534,7 +1534,7 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX)) + mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY)) if not mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return @@ -1552,8 +1552,8 @@ def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, A params = parse_qs(parsed.query) repo = str(payload.get("source_repository") or params.get("repo", [""])[0] or claims.get("repository") or "unknown") task = str(payload.get("task") or params.get("task", [""])[0] or "") - raw_mode = payload.get("mode") if "mode" in payload else params.get("mode", [MODE_REVIEW_AND_FIX])[0] - requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_AND_FIX)) + raw_mode = payload.get("mode") if "mode" in payload else params.get("mode", [MODE_REVIEW_ONLY])[0] + requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_ONLY)) if not requested_mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return @@ -1646,7 +1646,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st if mode_from_payload else existing_metadata.get("requested_mode") or existing_metadata.get("mode") ) - default_mode = MODE_REVIEW_AND_FIX + default_mode = MODE_REVIEW_ONLY requested_mode = _normalize_control_mode_param(str(raw_mode or default_mode)) if mode_from_payload and not requested_mode: raise ValueError("invalid mode") diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index df6a803d..1398a7b0 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -13,7 +13,7 @@ class TestAutomationControlSnapshot(unittest.TestCase): - def test_control_snapshot_defaults_to_review_and_fix_for_healthy_repo(self) -> None: + def test_control_snapshot_defaults_to_review_only_for_healthy_repo(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -27,9 +27,9 @@ def test_control_snapshot_defaults_to_review_and_fix_for_healthy_repo(self) -> N ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") - self.assertEqual(control["action"], "continue") - self.assertEqual(control["execution"]["effective_mode"], "review_and_fix") - self.assertTrue(control["execution"]["auto_fix_allowed"]) + self.assertEqual(control["action"], "review_only") + self.assertEqual(control["execution"]["effective_mode"], "review_only") + self.assertFalse(control["execution"]["auto_fix_allowed"]) def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> None: health = type("Health", (), {"status": "healthy"})() @@ -64,7 +64,7 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: }, "repositories": { "QuantStrategyLab/TargetRepo": { - "max_autonomy": "review_only", + "max_autonomy": "manual", "max_consecutive_failures": 2, } } @@ -94,7 +94,8 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: self.assertEqual(control["action"], "review_only") self.assertEqual(control["effective_action"], "review_only") - self.assertFalse(control["requires_human_review"]) + self.assertTrue(control["requires_human_review"]) + self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) @@ -143,8 +144,8 @@ def snapshot(self, limit=100): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") self.assertIsNone(ledger.requested_limit) - self.assertEqual(control["action"], "escalate") - self.assertEqual(control["effective_action"], "escalate") + self.assertEqual(control["action"], "review_only") + self.assertEqual(control["effective_action"], "review_only") self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) @@ -181,8 +182,8 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None pending_run=pending_run, ) - self.assertEqual(control["action"], "escalate") - self.assertEqual(control["effective_action"], "escalate") + self.assertEqual(control["action"], "review_only") + self.assertEqual(control["effective_action"], "review_only") self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) @@ -223,8 +224,8 @@ def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "escalate") - self.assertEqual(control["effective_action"], "escalate") + self.assertEqual(control["action"], "review_only") + self.assertEqual(control["effective_action"], "review_only") self.assertEqual(control["execution"]["action"], "human_review") self.assertFalse(control["execution"]["failure_history_complete"]) self.assertEqual(control["execution"]["consecutive_failures"], 1) @@ -258,7 +259,7 @@ def test_control_snapshot_ignores_other_repo_ledger_eviction(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "continue") + self.assertEqual(control["action"], "review_only") self.assertTrue(control["execution"]["failure_history_complete"]) def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> None: @@ -284,8 +285,8 @@ def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> N ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "escalate") - self.assertEqual(control["effective_action"], "escalate") + self.assertEqual(control["action"], "review_only") + self.assertEqual(control["effective_action"], "review_only") self.assertFalse(control["execution"]["failure_history_complete"]) def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(self) -> None: @@ -334,7 +335,7 @@ def test_triage_does_not_allow_auto_fix_when_execution_disallows_it(self) -> Non self.assertEqual(triage["recommended_action"], "open_issue") self.assertEqual(triage["next_step"], "open_issue") - def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: + def test_triage_omitted_mode_keeps_review_only_default(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -352,9 +353,9 @@ def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: changed_paths=["docs/runbook.md"], ) - self.assertEqual(triage["control"]["execution"]["requested_mode"], "review_and_fix") - self.assertTrue(triage["auto_fix_allowed"]) - self.assertEqual(triage["recommended_action"], "open_fix_pr") + self.assertEqual(triage["control"]["execution"]["requested_mode"], "review_only") + self.assertFalse(triage["auto_fix_allowed"]) + self.assertEqual(triage["recommended_action"], "open_issue") def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: runs = [ diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index a6104a2a..74067242 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -375,7 +375,7 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None control = json.loads(response.read().decode("utf-8"))["control"] self.assertIn("execution", control) self.assertEqual(control["execution"]["repo"], "QuantStrategyLab/TargetRepo") - self.assertEqual(control["execution"]["requested_mode"], "review_and_fix") + self.assertEqual(control["execution"]["requested_mode"], "review_only") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) @@ -583,8 +583,8 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: recorded = json.loads(response.read().decode("utf-8")) self.assertEqual(recorded["run"]["run_id"], "platform-health-run-1") self.assertEqual(recorded["run"]["suggested_action"], recorded["control"]["effective_action"]) - self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_and_fix") - self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_and_fix") + self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_only") + self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_only") self.assertEqual(recorded["run"]["service_health"], recorded["control"]["service_health"]) self.assertEqual(recorded["run"]["quota_status"], recorded["control"]["quota_status"]) self.assertEqual(recorded["run"]["org_health_status"], recorded["control"]["org_health_status"]) From ce374cab325ef81e2f1347ca59b8be749c40d7ac Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 07:58:24 +0800 Subject: [PATCH 43/72] fix: preserve automation defaults with fresh run history Co-Authored-By: Codex --- service/ai_gateway_service.py | 51 +++++++++++---------- tests/test_ai_gateway_automation_control.py | 18 ++++---- tests/test_ai_gateway_service_get_routes.py | 6 +-- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 20d5c256..3098602b 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -513,7 +513,7 @@ def _automation_control_snapshot( repo: str, *, task_name: str = "", - requested_mode: str = MODE_REVIEW_ONLY, + requested_mode: str = MODE_REVIEW_AND_FIX, pending_run: dict[str, Any] | None = None, ) -> dict[str, Any]: try: @@ -530,34 +530,35 @@ def _automation_control_snapshot( recent_runs = ledger_snapshot["runs"] ledger_summary = ledger_snapshot.get("summary") if isinstance(ledger_snapshot.get("summary"), dict) else {} retention = ledger_summary.get("retention") if isinstance(ledger_summary.get("retention"), dict) else {} - evicted_by_repo = ( - retention.get("evicted_runs_by_repo") if isinstance(retention.get("evicted_runs_by_repo"), dict) else {} - ) - try: - repo_evictions = int(evicted_by_repo.get(str(repo or "unknown").strip().lower(), 0) or 0) - except (TypeError, ValueError): - repo_evictions = 0 - normalized_repo = str(repo or "unknown").strip().lower() - repo_history_has_terminal_boundary = any( - str(run.get("task_state") or "").strip().lower() not in {"failed", "queued", "running", "pending", "in_progress"} - and _automation_run_owner_repository(run).strip().lower() == normalized_repo - for run in recent_runs - if isinstance(run, dict) - ) - failure_history_complete = ( - not bool(retention.get("history_completeness_unknown")) - and (repo_evictions <= 0 or repo_history_has_terminal_boundary) - ) ledger_unavailable = False except Exception: recent_runs = [] - failure_history_complete = False + retention = {} ledger_unavailable = True if pending_run is not None: pending_run_id = str(pending_run.get("run_id") or "") if pending_run_id: recent_runs = [run for run in recent_runs if str(run.get("run_id") or "") != pending_run_id] recent_runs = [pending_run, *recent_runs] + evicted_by_repo = ( + retention.get("evicted_runs_by_repo") if isinstance(retention.get("evicted_runs_by_repo"), dict) else {} + ) + try: + repo_evictions = int(evicted_by_repo.get(str(repo or "unknown").strip().lower(), 0) or 0) + except (TypeError, ValueError): + repo_evictions = 0 + normalized_repo = str(repo or "unknown").strip().lower() + repo_history_has_terminal_boundary = any( + str(run.get("task_state") or "").strip().lower() not in {"failed", "queued", "running", "pending", "in_progress"} + and _automation_run_owner_repository(run).strip().lower() == normalized_repo + for run in recent_runs + if isinstance(run, dict) + ) + failure_history_complete = ( + not ledger_unavailable + and not bool(retention.get("history_completeness_unknown")) + and (repo_evictions <= 0 or repo_history_has_terminal_boundary) + ) execution = decide_automation_execution( repo=repo or "unknown", task_name=task_name, @@ -627,7 +628,7 @@ def _automation_triage_snapshot( repo: str, *, task: str = "", - requested_mode: str = MODE_REVIEW_ONLY, + requested_mode: str = MODE_REVIEW_AND_FIX, failure_category: str = "", error: str = "", changed_paths: list[str] | None = None, @@ -1534,7 +1535,7 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY)) + mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX)) if not mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return @@ -1552,8 +1553,8 @@ def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, A params = parse_qs(parsed.query) repo = str(payload.get("source_repository") or params.get("repo", [""])[0] or claims.get("repository") or "unknown") task = str(payload.get("task") or params.get("task", [""])[0] or "") - raw_mode = payload.get("mode") if "mode" in payload else params.get("mode", [MODE_REVIEW_ONLY])[0] - requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_ONLY)) + raw_mode = payload.get("mode") if "mode" in payload else params.get("mode", [MODE_REVIEW_AND_FIX])[0] + requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_AND_FIX)) if not requested_mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return @@ -1646,7 +1647,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st if mode_from_payload else existing_metadata.get("requested_mode") or existing_metadata.get("mode") ) - default_mode = MODE_REVIEW_ONLY + default_mode = MODE_REVIEW_AND_FIX requested_mode = _normalize_control_mode_param(str(raw_mode or default_mode)) if mode_from_payload and not requested_mode: raise ValueError("invalid mode") diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 1398a7b0..973cfa8b 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -13,7 +13,7 @@ class TestAutomationControlSnapshot(unittest.TestCase): - def test_control_snapshot_defaults_to_review_only_for_healthy_repo(self) -> None: + def test_control_snapshot_defaults_to_review_and_fix_for_healthy_repo(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -27,9 +27,9 @@ def test_control_snapshot_defaults_to_review_only_for_healthy_repo(self) -> None ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") - self.assertEqual(control["action"], "review_only") - self.assertEqual(control["execution"]["effective_mode"], "review_only") - self.assertFalse(control["execution"]["auto_fix_allowed"]) + self.assertEqual(control["action"], "continue") + self.assertEqual(control["execution"]["effective_mode"], "review_and_fix") + self.assertTrue(control["execution"]["auto_fix_allowed"]) def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> None: health = type("Health", (), {"status": "healthy"})() @@ -259,7 +259,7 @@ def test_control_snapshot_ignores_other_repo_ledger_eviction(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "review_only") + self.assertEqual(control["action"], "continue") self.assertTrue(control["execution"]["failure_history_complete"]) def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> None: @@ -335,7 +335,7 @@ def test_triage_does_not_allow_auto_fix_when_execution_disallows_it(self) -> Non self.assertEqual(triage["recommended_action"], "open_issue") self.assertEqual(triage["next_step"], "open_issue") - def test_triage_omitted_mode_keeps_review_only_default(self) -> None: + def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -353,9 +353,9 @@ def test_triage_omitted_mode_keeps_review_only_default(self) -> None: changed_paths=["docs/runbook.md"], ) - self.assertEqual(triage["control"]["execution"]["requested_mode"], "review_only") - self.assertFalse(triage["auto_fix_allowed"]) - self.assertEqual(triage["recommended_action"], "open_issue") + self.assertEqual(triage["control"]["execution"]["requested_mode"], "review_and_fix") + self.assertTrue(triage["auto_fix_allowed"]) + self.assertEqual(triage["recommended_action"], "open_fix_pr") def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: runs = [ diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 74067242..a6104a2a 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -375,7 +375,7 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None control = json.loads(response.read().decode("utf-8"))["control"] self.assertIn("execution", control) self.assertEqual(control["execution"]["repo"], "QuantStrategyLab/TargetRepo") - self.assertEqual(control["execution"]["requested_mode"], "review_only") + self.assertEqual(control["execution"]["requested_mode"], "review_and_fix") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) @@ -583,8 +583,8 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: recorded = json.loads(response.read().decode("utf-8")) self.assertEqual(recorded["run"]["run_id"], "platform-health-run-1") self.assertEqual(recorded["run"]["suggested_action"], recorded["control"]["effective_action"]) - self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_only") - self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_only") + self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_and_fix") + self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_and_fix") self.assertEqual(recorded["run"]["service_health"], recorded["control"]["service_health"]) self.assertEqual(recorded["run"]["quota_status"], recorded["control"]["quota_status"]) self.assertEqual(recorded["run"]["org_health_status"], recorded["control"]["org_health_status"]) From b7ce2175c89305e9fbc921d78200286a2855f451 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:07:26 +0800 Subject: [PATCH 44/72] fix: keep human review legacy escalation Co-Authored-By: Codex --- service/ai_gateway_service.py | 5 +++-- tests/test_ai_gateway_automation_control.py | 20 +++++++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 3098602b..f2d5d43e 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -584,7 +584,7 @@ def _automation_control_snapshot( original_action = str(control.get("action") or CONTROL_REVIEW_ONLY) strict_action = original_action if execution.get("action") == EXECUTION_HUMAN_REVIEW: - strict_action = CONTROL_ESCALATE if original_action == CONTROL_ESCALATE or ledger_unavailable else CONTROL_REVIEW_ONLY + strict_action = CONTROL_ESCALATE elif execution.get("action") == EXECUTION_REVIEW_ONLY and strict_action != CONTROL_PAUSE_AUTO_FIX: strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: @@ -649,6 +649,7 @@ def _automation_triage_snapshot( category = service_failure_category(error) control_action = str(control.get("effective_action") or control.get("action") or CONTROL_REVIEW_ONLY) + runtime_control_action = str(control.get("runtime_action") or control.get("action") or CONTROL_REVIEW_ONLY) execution = control.get("execution") if isinstance(control.get("execution"), dict) else {} execution_auto_fix_allowed = bool(control.get("auto_fix_allowed")) and bool(execution.get("auto_fix_allowed")) retry_allowed = False @@ -674,7 +675,7 @@ def _automation_triage_snapshot( recommended_action = "retry" next_step = "retry" else: - if path_risk in {RISK_CRITICAL, RISK_HIGH} or control_action == CONTROL_ESCALATE: + if path_risk in {RISK_CRITICAL, RISK_HIGH} or runtime_control_action == CONTROL_ESCALATE: incident_class = "blocked" recommended_action = "escalate" next_step = "escalate" diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 973cfa8b..21989e4b 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -92,12 +92,11 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") - self.assertEqual(control["action"], "review_only") - self.assertEqual(control["effective_action"], "review_only") + self.assertEqual(control["action"], "escalate") + self.assertEqual(control["effective_action"], "escalate") self.assertTrue(control["requires_human_review"]) self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["effective_mode"], "review_only") - self.assertFalse(control["execution"]["auto_fix_allowed"]) def test_control_snapshot_scans_full_retained_ledger_for_repo_failure_streak(self) -> None: runs = [ @@ -144,8 +143,7 @@ def snapshot(self, limit=100): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") self.assertIsNone(ledger.requested_limit) - self.assertEqual(control["action"], "review_only") - self.assertEqual(control["effective_action"], "review_only") + self.assertEqual(control["action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) @@ -182,8 +180,8 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None pending_run=pending_run, ) - self.assertEqual(control["action"], "review_only") - self.assertEqual(control["effective_action"], "review_only") + self.assertEqual(control["action"], "escalate") + self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) @@ -224,8 +222,8 @@ def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "review_only") - self.assertEqual(control["effective_action"], "review_only") + self.assertEqual(control["action"], "escalate") + self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertFalse(control["execution"]["failure_history_complete"]) self.assertEqual(control["execution"]["consecutive_failures"], 1) @@ -285,8 +283,8 @@ def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> N ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "review_only") - self.assertEqual(control["effective_action"], "review_only") + self.assertEqual(control["action"], "escalate") + self.assertEqual(control["effective_action"], "escalate") self.assertFalse(control["execution"]["failure_history_complete"]) def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(self) -> None: From b800aff26de7c499e52911d6f68110899f9a8f0c Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:14:54 +0800 Subject: [PATCH 45/72] fix: infer legacy ledger completeness Co-Authored-By: Codex --- service/ai_gateway_service.py | 3 +-- service/automation_run_ledger.py | 2 +- tests/test_automation_run_ledger.py | 10 +++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index f2d5d43e..ea239eab 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -649,7 +649,6 @@ def _automation_triage_snapshot( category = service_failure_category(error) control_action = str(control.get("effective_action") or control.get("action") or CONTROL_REVIEW_ONLY) - runtime_control_action = str(control.get("runtime_action") or control.get("action") or CONTROL_REVIEW_ONLY) execution = control.get("execution") if isinstance(control.get("execution"), dict) else {} execution_auto_fix_allowed = bool(control.get("auto_fix_allowed")) and bool(execution.get("auto_fix_allowed")) retry_allowed = False @@ -675,7 +674,7 @@ def _automation_triage_snapshot( recommended_action = "retry" next_step = "retry" else: - if path_risk in {RISK_CRITICAL, RISK_HIGH} or runtime_control_action == CONTROL_ESCALATE: + if path_risk in {RISK_CRITICAL, RISK_HIGH} or control_action == CONTROL_ESCALATE: incident_class = "blocked" recommended_action = "escalate" next_step = "escalate" diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 2cd32b61..40712ea8 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -252,7 +252,7 @@ def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, return {}, 0, 0, {}, True clean_runs = {str(key): value for key, value in runs.items() if isinstance(value, dict)} sequence = _safe_int(payload.get("sequence"), len(clean_runs)) - history_unknown = "evicted_runs" not in payload or "evicted_runs_by_repo" not in payload + history_unknown = ("evicted_runs" not in payload or "evicted_runs_by_repo" not in payload) and len(clean_runs) >= self._max_runs evicted_runs = _safe_int(payload.get("evicted_runs"), 0) raw_evicted_by_repo = payload.get("evicted_runs_by_repo") evicted_by_repo = { diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index 7395b55c..1d7a4782 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -222,7 +222,7 @@ def test_persist_merge_does_not_recount_disk_evicted_local_runs(self) -> None: self.assertEqual(snapshot["summary"]["retention"]["evicted_runs_by_repo"], {"quantstrategylab/repoa": 2}) self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-3", "run-4"}) - def test_pre_migration_ledger_marks_history_completeness_unknown(self) -> None: + def test_pre_migration_full_ledger_marks_history_completeness_unknown(self) -> None: with TemporaryDirectory() as tmp: path = Path(tmp) / "automation_runs.json" path.write_text( @@ -243,12 +243,12 @@ def test_pre_migration_ledger_marks_history_completeness_unknown(self) -> None: encoding="utf-8", ) - ledger = AutomationRunLedger(max_runs=2, storage_path=path) + ledger = AutomationRunLedger(max_runs=1, storage_path=path) snapshot = ledger.snapshot(limit=None) self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) - def test_pre_migration_ledger_keeps_history_unknown_after_schema_rewrite(self) -> None: + def test_pre_migration_partial_ledger_clears_history_unknown_after_schema_rewrite(self) -> None: with TemporaryDirectory() as tmp: path = Path(tmp) / "automation_runs.json" path.write_text( @@ -270,11 +270,11 @@ def test_pre_migration_ledger_keeps_history_unknown_after_schema_rewrite(self) - ) ledger = AutomationRunLedger(max_runs=3, storage_path=path) - self.assertTrue(ledger.snapshot(limit=None)["summary"]["retention"]["history_completeness_unknown"]) + self.assertFalse(ledger.snapshot(limit=None)["summary"]["retention"]["history_completeness_unknown"]) ledger.record("run-2", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) snapshot = AutomationRunLedger(max_runs=3, storage_path=path).snapshot(limit=None) - self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) + self.assertFalse(snapshot["summary"]["retention"]["history_completeness_unknown"]) self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-1", "run-2"}) def test_stale_update_cannot_resurrect_run_evicted_on_disk(self) -> None: From 0d9ff1eac2d55b35293fe944ee08aeae84a2a163 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:21:35 +0800 Subject: [PATCH 46/72] fix: fail closed on autonomy caps Co-Authored-By: Codex --- service/ai_gateway_service.py | 2 ++ tests/test_ai_gateway_automation_control.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index ea239eab..e65413b2 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -589,6 +589,8 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX + elif execution.get("requested_autonomy") != execution.get("effective_autonomy") and strict_action == CONTROL_CONTINUE: + strict_action = CONTROL_REVIEW_ONLY elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: strict_action = CONTROL_REVIEW_ONLY control["runtime_action"] = original_action diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 21989e4b..6a1dd5aa 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -287,7 +287,7 @@ def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> N self.assertEqual(control["effective_action"], "escalate") self.assertFalse(control["execution"]["failure_history_complete"]) - def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(self) -> None: + def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -301,8 +301,8 @@ def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(se ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") - self.assertEqual(control["action"], "continue") - self.assertTrue(control["auto_fix_allowed"]) + self.assertEqual(control["action"], "review_only") + self.assertFalse(control["auto_fix_allowed"]) self.assertFalse(control["auto_merge_allowed"]) self.assertEqual(control["execution"]["requested_autonomy"], "auto_merge") self.assertEqual(control["execution"]["effective_autonomy"], "auto_pr") From 86e3ac475bc3b2981310cecefc5494879c69232f Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:26:39 +0800 Subject: [PATCH 47/72] fix: keep legacy automation compatibility Co-Authored-By: Codex --- service/ai_gateway_service.py | 2 -- service/automation_run_ledger.py | 2 +- tests/test_ai_gateway_automation_control.py | 6 +++--- tests/test_automation_run_ledger.py | 10 +++++----- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index e65413b2..ea239eab 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -589,8 +589,6 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX - elif execution.get("requested_autonomy") != execution.get("effective_autonomy") and strict_action == CONTROL_CONTINUE: - strict_action = CONTROL_REVIEW_ONLY elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: strict_action = CONTROL_REVIEW_ONLY control["runtime_action"] = original_action diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 40712ea8..2cd32b61 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -252,7 +252,7 @@ def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, return {}, 0, 0, {}, True clean_runs = {str(key): value for key, value in runs.items() if isinstance(value, dict)} sequence = _safe_int(payload.get("sequence"), len(clean_runs)) - history_unknown = ("evicted_runs" not in payload or "evicted_runs_by_repo" not in payload) and len(clean_runs) >= self._max_runs + history_unknown = "evicted_runs" not in payload or "evicted_runs_by_repo" not in payload evicted_runs = _safe_int(payload.get("evicted_runs"), 0) raw_evicted_by_repo = payload.get("evicted_runs_by_repo") evicted_by_repo = { diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 6a1dd5aa..21989e4b 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -287,7 +287,7 @@ def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> N self.assertEqual(control["effective_action"], "escalate") self.assertFalse(control["execution"]["failure_history_complete"]) - def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(self) -> None: + def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -301,8 +301,8 @@ def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(sel ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") - self.assertEqual(control["action"], "review_only") - self.assertFalse(control["auto_fix_allowed"]) + self.assertEqual(control["action"], "continue") + self.assertTrue(control["auto_fix_allowed"]) self.assertFalse(control["auto_merge_allowed"]) self.assertEqual(control["execution"]["requested_autonomy"], "auto_merge") self.assertEqual(control["execution"]["effective_autonomy"], "auto_pr") diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index 1d7a4782..7395b55c 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -222,7 +222,7 @@ def test_persist_merge_does_not_recount_disk_evicted_local_runs(self) -> None: self.assertEqual(snapshot["summary"]["retention"]["evicted_runs_by_repo"], {"quantstrategylab/repoa": 2}) self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-3", "run-4"}) - def test_pre_migration_full_ledger_marks_history_completeness_unknown(self) -> None: + def test_pre_migration_ledger_marks_history_completeness_unknown(self) -> None: with TemporaryDirectory() as tmp: path = Path(tmp) / "automation_runs.json" path.write_text( @@ -243,12 +243,12 @@ def test_pre_migration_full_ledger_marks_history_completeness_unknown(self) -> N encoding="utf-8", ) - ledger = AutomationRunLedger(max_runs=1, storage_path=path) + ledger = AutomationRunLedger(max_runs=2, storage_path=path) snapshot = ledger.snapshot(limit=None) self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) - def test_pre_migration_partial_ledger_clears_history_unknown_after_schema_rewrite(self) -> None: + def test_pre_migration_ledger_keeps_history_unknown_after_schema_rewrite(self) -> None: with TemporaryDirectory() as tmp: path = Path(tmp) / "automation_runs.json" path.write_text( @@ -270,11 +270,11 @@ def test_pre_migration_partial_ledger_clears_history_unknown_after_schema_rewrit ) ledger = AutomationRunLedger(max_runs=3, storage_path=path) - self.assertFalse(ledger.snapshot(limit=None)["summary"]["retention"]["history_completeness_unknown"]) + self.assertTrue(ledger.snapshot(limit=None)["summary"]["retention"]["history_completeness_unknown"]) ledger.record("run-2", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) snapshot = AutomationRunLedger(max_runs=3, storage_path=path).snapshot(limit=None) - self.assertFalse(snapshot["summary"]["retention"]["history_completeness_unknown"]) + self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-1", "run-2"}) def test_stale_update_cannot_resurrect_run_evicted_on_disk(self) -> None: From 1038fa9252b090640fd732b17a1683c12a02988f Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:35:25 +0800 Subject: [PATCH 48/72] fix: guard capped merges and stale ledger updates Co-Authored-By: Codex --- service/ai_gateway_service.py | 2 ++ service/automation_run_ledger.py | 4 +--- tests/test_ai_gateway_automation_control.py | 6 +++--- tests/test_automation_run_ledger.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index ea239eab..80eaed61 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -589,6 +589,8 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX + elif execution.get("requested_autonomy") == "auto_merge" and execution.get("effective_autonomy") != "auto_merge": + strict_action = CONTROL_REVIEW_ONLY elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: strict_action = CONTROL_REVIEW_ONLY control["runtime_action"] = original_action diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 2cd32b61..70bb1fe7 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -271,8 +271,6 @@ def _merge_evicted_runs_by_repo_locked(self, disk_evicted_runs_by_repo: dict[str self._evicted_runs_by_repo[repo_key] = max(self._evicted_runs_by_repo.get(repo_key, 0), int(count)) def _drop_runs_evicted_on_disk_locked(self, disk_runs: dict[str, dict[str, Any]], *, preserve_run_id: str = "") -> None: - if len(disk_runs) < self._max_runs: - return disk_run_ids = set(disk_runs) for run_id in list(self._runs): if run_id != preserve_run_id and run_id not in disk_run_ids: @@ -350,7 +348,7 @@ def _persist_with_owner_guard_locked( disk_owner = _entry_owner_repository(disk_entry) if isinstance(disk_entry, dict) else "" if owner_repository and disk_owner and disk_owner != owner_repository: raise PermissionError("automation run_id belongs to another repository") - if guard_preexisting and disk_entry is None and len(disk_runs) >= self._max_runs: + if guard_preexisting and disk_entry is None and self._storage_file_seen: raise ValueError("automation run was evicted from retained ledger") self._drop_runs_evicted_on_disk_locked(disk_runs, preserve_run_id=guard_run_id) for run_id, disk_entry in disk_runs.items(): diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 21989e4b..6a1dd5aa 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -287,7 +287,7 @@ def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> N self.assertEqual(control["effective_action"], "escalate") self.assertFalse(control["execution"]["failure_history_complete"]) - def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(self) -> None: + def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -301,8 +301,8 @@ def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(se ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") - self.assertEqual(control["action"], "continue") - self.assertTrue(control["auto_fix_allowed"]) + self.assertEqual(control["action"], "review_only") + self.assertFalse(control["auto_fix_allowed"]) self.assertFalse(control["auto_merge_allowed"]) self.assertEqual(control["execution"]["requested_autonomy"], "auto_merge") self.assertEqual(control["execution"]["effective_autonomy"], "auto_pr") diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index 7395b55c..dc64a045 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -284,7 +284,7 @@ def test_stale_update_cannot_resurrect_run_evicted_on_disk(self) -> None: with patch("service.automation_run_ledger.time.time", side_effect=[1.0, 2.0]): ledger_a.record("run-1", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) ledger_a.record("run-2", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) - ledger_b = AutomationRunLedger(max_runs=2, storage_path=path) + ledger_b = AutomationRunLedger(max_runs=3, storage_path=path) with patch("service.automation_run_ledger.time.time", return_value=3.0): ledger_a.record("run-3", "queued", metadata={"source_repository": "QuantStrategyLab/RepoA"}) with patch("service.automation_run_ledger.time.time", return_value=4.0): From af403918b388f0a8db890c318a92301af3c26bae Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:49:12 +0800 Subject: [PATCH 49/72] fix: address automation review blockers Co-Authored-By: Codex --- service/ai_gateway_service.py | 2 -- service/automation_run_ledger.py | 24 ++++++++++++----- tests/test_ai_gateway_automation_control.py | 29 +++------------------ tests/test_automation_run_ledger.py | 14 ++++++++++ 4 files changed, 34 insertions(+), 35 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 80eaed61..ea239eab 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -589,8 +589,6 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX - elif execution.get("requested_autonomy") == "auto_merge" and execution.get("effective_autonomy") != "auto_merge": - strict_action = CONTROL_REVIEW_ONLY elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: strict_action = CONTROL_REVIEW_ONLY control["runtime_action"] = original_action diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 70bb1fe7..842bedbd 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -230,7 +230,7 @@ def _load_from_disk(self) -> None: return if not self._storage_path.exists(): return - runs, sequence, evicted_runs, evicted_runs_by_repo, history_unknown = self._read_from_disk_unlocked() + runs, sequence, evicted_runs, evicted_runs_by_repo, history_unknown, _read_ok = self._read_from_disk_unlocked() with self._lock: self._storage_file_seen = True self._runs = runs @@ -240,16 +240,16 @@ def _load_from_disk(self) -> None: self._history_completeness_unknown = history_unknown self._evict_old_runs_locked() - def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, dict[str, int], bool]: + def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, dict[str, int], bool, bool]: if self._storage_path is None or not self._storage_path.exists(): - return {}, 0, 0, {}, self._storage_file_seen + return {}, 0, 0, {}, self._storage_file_seen, not self._storage_file_seen try: payload = json.loads(self._storage_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): - return {}, 0, 0, {}, True + return {}, 0, 0, {}, True, False runs = payload.get("runs") if isinstance(payload, dict) else None if not isinstance(runs, dict): - return {}, 0, 0, {}, True + return {}, 0, 0, {}, True, False clean_runs = {str(key): value for key, value in runs.items() if isinstance(value, dict)} sequence = _safe_int(payload.get("sequence"), len(clean_runs)) history_unknown = "evicted_runs" not in payload or "evicted_runs_by_repo" not in payload @@ -261,7 +261,7 @@ def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, if _repo_retention_key(repo) } history_unknown = bool(payload.get("history_completeness_unknown", history_unknown)) - return clean_runs, sequence, max(0, evicted_runs), evicted_by_repo, history_unknown + return clean_runs, sequence, max(0, evicted_runs), evicted_by_repo, history_unknown, True def _merge_evicted_runs_by_repo_locked(self, disk_evicted_runs_by_repo: dict[str, int]) -> None: for repo, count in disk_evicted_runs_by_repo.items(): @@ -298,8 +298,12 @@ def _refresh_from_disk_locked(self) -> None: disk_evicted_runs, disk_evicted_runs_by_repo, disk_history_unknown, + disk_read_ok, ) = self._read_from_disk_unlocked() self._storage_file_seen = True + if not disk_read_ok: + self._history_completeness_unknown = True + return self._drop_runs_evicted_on_disk_locked(disk_runs) for run_id, disk_entry in disk_runs.items(): current = self._runs.get(run_id) @@ -335,20 +339,26 @@ def _persist_with_owner_guard_locked( lock_path = self._storage_path.with_suffix(self._storage_path.suffix + ".lock") lock_handle = lock_path.open("a+", encoding="utf-8") fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) + had_seen_storage = self._storage_file_seen ( disk_runs, disk_sequence, disk_evicted_runs, disk_evicted_runs_by_repo, disk_history_unknown, + disk_read_ok, ) = self._read_from_disk_unlocked() self._storage_file_seen = True + if not disk_read_ok: + self._history_completeness_unknown = True + self._evict_old_runs_locked() + return if guard_run_id: disk_entry = disk_runs.get(guard_run_id) disk_owner = _entry_owner_repository(disk_entry) if isinstance(disk_entry, dict) else "" if owner_repository and disk_owner and disk_owner != owner_repository: raise PermissionError("automation run_id belongs to another repository") - if guard_preexisting and disk_entry is None and self._storage_file_seen: + if guard_preexisting and disk_entry is None and had_seen_storage: raise ValueError("automation run was evicted from retained ledger") self._drop_runs_evicted_on_disk_locked(disk_runs, preserve_run_id=guard_run_id) for run_id, disk_entry in disk_runs.items(): diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 6a1dd5aa..85ec6da2 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -287,7 +287,7 @@ def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> N self.assertEqual(control["effective_action"], "escalate") self.assertFalse(control["execution"]["failure_history_complete"]) - def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(self) -> None: + def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -301,8 +301,8 @@ def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(sel ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") - self.assertEqual(control["action"], "review_only") - self.assertFalse(control["auto_fix_allowed"]) + self.assertEqual(control["action"], "continue") + self.assertTrue(control["auto_fix_allowed"]) self.assertFalse(control["auto_merge_allowed"]) self.assertEqual(control["execution"]["requested_autonomy"], "auto_merge") self.assertEqual(control["execution"]["effective_autonomy"], "auto_pr") @@ -310,29 +310,6 @@ def test_control_snapshot_downgrades_legacy_action_when_auto_merge_is_capped(sel self.assertTrue(control["execution"]["auto_fix_allowed"]) self.assertFalse(control["execution"]["auto_merge_allowed"]) - def test_triage_does_not_allow_auto_fix_when_execution_disallows_it(self) -> None: - control = { - "action": "continue", - "auto_fix_allowed": False, - "requires_human_review": False, - "service_health": "healthy", - "quota_status": "ok", - "org_health_status": "ok", - "execution": {"auto_fix_allowed": False}, - } - - with patch("service.ai_gateway_service._automation_control_snapshot", return_value=control): - triage = _automation_triage_snapshot( - "QuantStrategyLab/TargetRepo", - task="monthly", - changed_paths=["docs/runbook.md"], - ) - - self.assertFalse(triage["auto_fix_allowed"]) - self.assertFalse(triage["deploy_allowed"]) - self.assertEqual(triage["recommended_action"], "open_issue") - self.assertEqual(triage["next_step"], "open_issue") - def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index dc64a045..6b15a3ec 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -320,6 +320,20 @@ def test_corrupt_persisted_ledger_marks_history_completeness_unknown(self) -> No self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) + def test_corrupt_disk_read_preserves_local_runs_without_overwriting_file(self) -> None: + with TemporaryDirectory() as tmp: + path = Path(tmp) / "automation_runs.json" + ledger = AutomationRunLedger(max_runs=3, storage_path=path) + ledger.record("run-1", "queued") + path.write_text("{not-json", encoding="utf-8") + ledger.record("run-2", "queued") + snapshot = ledger.snapshot(limit=None) + + self.assertEqual(path.read_text(encoding="utf-8"), "{not-json") + + self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-1", "run-2"}) + self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) + def test_update_preserves_control_fields_when_omitted(self) -> None: self.ledger.record( "run-1", From 38e35761d915c15783294181fce3a0971db7a840 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:56:50 +0800 Subject: [PATCH 50/72] fix: scope ledger history safety checks Co-Authored-By: Codex --- service/ai_gateway_service.py | 2 +- service/automation_run_ledger.py | 3 +-- tests/test_ai_gateway_automation_control.py | 7 +++++-- tests/test_automation_run_ledger.py | 7 ++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index ea239eab..48c2db71 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -556,7 +556,7 @@ def _automation_control_snapshot( ) failure_history_complete = ( not ledger_unavailable - and not bool(retention.get("history_completeness_unknown")) + and (not bool(retention.get("history_completeness_unknown")) or repo_history_has_terminal_boundary) and (repo_evictions <= 0 or repo_history_has_terminal_boundary) ) execution = decide_automation_execution( diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 842bedbd..1c2806f5 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -351,8 +351,7 @@ def _persist_with_owner_guard_locked( self._storage_file_seen = True if not disk_read_ok: self._history_completeness_unknown = True - self._evict_old_runs_locked() - return + raise OSError("automation ledger could not be refreshed from disk") if guard_run_id: disk_entry = disk_runs.get(guard_run_id) disk_owner = _entry_owner_repository(disk_entry) if isinstance(disk_entry, dict) else "" diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 85ec6da2..aabcb3a2 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -228,7 +228,7 @@ def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: self.assertFalse(control["execution"]["failure_history_complete"]) self.assertEqual(control["execution"]["consecutive_failures"], 1) - def test_control_snapshot_ignores_other_repo_ledger_eviction(self) -> None: + def test_control_snapshot_allows_known_repo_boundary_when_history_unknown(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type( @@ -236,9 +236,12 @@ def test_control_snapshot_ignores_other_repo_ledger_eviction(self) -> None: (), { "snapshot": lambda self, limit=None: { - "runs": [], + "runs": [ + {"run_id": "merged-1", "task_state": "merged", "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}} + ], "summary": { "retention": { + "history_completeness_unknown": True, "may_be_truncated": True, "evicted_runs": 1, "evicted_runs_by_repo": {"quantstrategylab/otherrepo": 1}, diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index 6b15a3ec..81a25b3b 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -320,18 +320,19 @@ def test_corrupt_persisted_ledger_marks_history_completeness_unknown(self) -> No self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) - def test_corrupt_disk_read_preserves_local_runs_without_overwriting_file(self) -> None: + def test_corrupt_disk_read_fails_record_without_overwriting_file(self) -> None: with TemporaryDirectory() as tmp: path = Path(tmp) / "automation_runs.json" ledger = AutomationRunLedger(max_runs=3, storage_path=path) ledger.record("run-1", "queued") path.write_text("{not-json", encoding="utf-8") - ledger.record("run-2", "queued") + with self.assertRaises(OSError): + ledger.record("run-2", "queued") snapshot = ledger.snapshot(limit=None) self.assertEqual(path.read_text(encoding="utf-8"), "{not-json") - self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-1", "run-2"}) + self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-1"}) self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) def test_update_preserves_control_fields_when_omitted(self) -> None: From 97cb0a95111b108858f7e7924a5dc58242ab3296 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:06:25 +0800 Subject: [PATCH 51/72] fix: align automation mode defaults Co-Authored-By: Codex --- service/ai_gateway_service.py | 14 ++++++------- tests/test_ai_gateway_automation_control.py | 22 ++++++++++----------- tests/test_ai_gateway_service_get_routes.py | 6 +++--- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 48c2db71..0b183274 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -513,7 +513,7 @@ def _automation_control_snapshot( repo: str, *, task_name: str = "", - requested_mode: str = MODE_REVIEW_AND_FIX, + requested_mode: str = MODE_REVIEW_ONLY, pending_run: dict[str, Any] | None = None, ) -> dict[str, Any]: try: @@ -600,7 +600,7 @@ def _automation_control_snapshot( control["reasons"] = reasons control["auto_fix_allowed"] = bool(execution.get("auto_fix_allowed")) and strict_action == CONTROL_CONTINUE control["auto_merge_allowed"] = bool(execution.get("auto_merge_allowed")) and strict_action == CONTROL_CONTINUE - control["requires_human_review"] = bool(execution.get("human_review_required")) + control["requires_human_review"] = strict_action != CONTROL_CONTINUE or bool(execution.get("human_review_required")) control["execution"] = execution return control @@ -628,7 +628,7 @@ def _automation_triage_snapshot( repo: str, *, task: str = "", - requested_mode: str = MODE_REVIEW_AND_FIX, + requested_mode: str = MODE_REVIEW_ONLY, failure_category: str = "", error: str = "", changed_paths: list[str] | None = None, @@ -1535,7 +1535,7 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX)) + mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY)) if not mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return @@ -1553,8 +1553,8 @@ def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, A params = parse_qs(parsed.query) repo = str(payload.get("source_repository") or params.get("repo", [""])[0] or claims.get("repository") or "unknown") task = str(payload.get("task") or params.get("task", [""])[0] or "") - raw_mode = payload.get("mode") if "mode" in payload else params.get("mode", [MODE_REVIEW_AND_FIX])[0] - requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_AND_FIX)) + raw_mode = payload.get("mode") if "mode" in payload else params.get("mode", [MODE_REVIEW_ONLY])[0] + requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_ONLY)) if not requested_mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return @@ -1647,7 +1647,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st if mode_from_payload else existing_metadata.get("requested_mode") or existing_metadata.get("mode") ) - default_mode = MODE_REVIEW_AND_FIX + default_mode = MODE_REVIEW_ONLY requested_mode = _normalize_control_mode_param(str(raw_mode or default_mode)) if mode_from_payload and not requested_mode: raise ValueError("invalid mode") diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index aabcb3a2..1e5f83e0 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -13,7 +13,7 @@ class TestAutomationControlSnapshot(unittest.TestCase): - def test_control_snapshot_defaults_to_review_and_fix_for_healthy_repo(self) -> None: + def test_control_snapshot_defaults_to_review_only_for_healthy_repo(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -27,9 +27,9 @@ def test_control_snapshot_defaults_to_review_and_fix_for_healthy_repo(self) -> N ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") - self.assertEqual(control["action"], "continue") - self.assertEqual(control["execution"]["effective_mode"], "review_and_fix") - self.assertTrue(control["execution"]["auto_fix_allowed"]) + self.assertEqual(control["action"], "review_only") + self.assertEqual(control["execution"]["effective_mode"], "review_only") + self.assertFalse(control["execution"]["auto_fix_allowed"]) def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> None: health = type("Health", (), {"status": "healthy"})() @@ -48,7 +48,7 @@ def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> self.assertEqual(control["action"], "review_only") self.assertEqual(control["effective_action"], "review_only") self.assertEqual(control["execution"]["effective_mode"], "review_only") - self.assertFalse(control["auto_fix_allowed"]) + self.assertTrue(control["requires_human_review"]) def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: with TemporaryDirectory(dir=".") as tmp: @@ -258,7 +258,7 @@ def test_control_snapshot_allows_known_repo_boundary_when_history_unknown(self) patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), patch("service.ai_gateway_service.load_execution_policy", return_value={"default": {"max_consecutive_failures": 2}}), ): - control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly", requested_mode="review_and_fix") self.assertEqual(control["action"], "continue") self.assertTrue(control["execution"]["failure_history_complete"]) @@ -313,7 +313,7 @@ def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(se self.assertTrue(control["execution"]["auto_fix_allowed"]) self.assertFalse(control["execution"]["auto_merge_allowed"]) - def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: + def test_triage_omitted_mode_uses_review_only_default(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -331,9 +331,9 @@ def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: changed_paths=["docs/runbook.md"], ) - self.assertEqual(triage["control"]["execution"]["requested_mode"], "review_and_fix") - self.assertTrue(triage["auto_fix_allowed"]) - self.assertEqual(triage["recommended_action"], "open_fix_pr") + self.assertEqual(triage["control"]["execution"]["requested_mode"], "review_only") + self.assertFalse(triage["auto_fix_allowed"]) + self.assertEqual(triage["recommended_action"], "open_issue") def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: runs = [ @@ -389,7 +389,7 @@ def test_control_snapshot_keeps_legacy_pause_for_defer(self) -> None: control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") self.assertEqual(control["action"], "pause_auto_fix") - self.assertFalse(control["requires_human_review"]) + self.assertTrue(control["requires_human_review"]) self.assertEqual(control["execution"]["action"], "defer") def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index a6104a2a..74067242 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -375,7 +375,7 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None control = json.loads(response.read().decode("utf-8"))["control"] self.assertIn("execution", control) self.assertEqual(control["execution"]["repo"], "QuantStrategyLab/TargetRepo") - self.assertEqual(control["execution"]["requested_mode"], "review_and_fix") + self.assertEqual(control["execution"]["requested_mode"], "review_only") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) @@ -583,8 +583,8 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: recorded = json.loads(response.read().decode("utf-8")) self.assertEqual(recorded["run"]["run_id"], "platform-health-run-1") self.assertEqual(recorded["run"]["suggested_action"], recorded["control"]["effective_action"]) - self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_and_fix") - self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_and_fix") + self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_only") + self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_only") self.assertEqual(recorded["run"]["service_health"], recorded["control"]["service_health"]) self.assertEqual(recorded["run"]["quota_status"], recorded["control"]["quota_status"]) self.assertEqual(recorded["run"]["org_health_status"], recorded["control"]["org_health_status"]) From a56ad4e832bc240313701b265e0944d2a5fac082 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:12:28 +0800 Subject: [PATCH 52/72] fix: preserve legacy automation defaults Co-Authored-By: Codex --- service/ai_gateway_service.py | 12 ++++++------ service/automation_decision.py | 2 +- tests/test_ai_gateway_automation_control.py | 16 ++++++++-------- tests/test_ai_gateway_service_get_routes.py | 6 +++--- tests/test_automation_decision.py | 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 0b183274..3dc2c78c 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -513,7 +513,7 @@ def _automation_control_snapshot( repo: str, *, task_name: str = "", - requested_mode: str = MODE_REVIEW_ONLY, + requested_mode: str = MODE_REVIEW_AND_FIX, pending_run: dict[str, Any] | None = None, ) -> dict[str, Any]: try: @@ -628,7 +628,7 @@ def _automation_triage_snapshot( repo: str, *, task: str = "", - requested_mode: str = MODE_REVIEW_ONLY, + requested_mode: str = MODE_REVIEW_AND_FIX, failure_category: str = "", error: str = "", changed_paths: list[str] | None = None, @@ -1535,7 +1535,7 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_ONLY])[0] or MODE_REVIEW_ONLY)) + mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX)) if not mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return @@ -1553,8 +1553,8 @@ def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, A params = parse_qs(parsed.query) repo = str(payload.get("source_repository") or params.get("repo", [""])[0] or claims.get("repository") or "unknown") task = str(payload.get("task") or params.get("task", [""])[0] or "") - raw_mode = payload.get("mode") if "mode" in payload else params.get("mode", [MODE_REVIEW_ONLY])[0] - requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_ONLY)) + raw_mode = payload.get("mode") if "mode" in payload else params.get("mode", [MODE_REVIEW_AND_FIX])[0] + requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_AND_FIX)) if not requested_mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return @@ -1647,7 +1647,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st if mode_from_payload else existing_metadata.get("requested_mode") or existing_metadata.get("mode") ) - default_mode = MODE_REVIEW_ONLY + default_mode = MODE_REVIEW_AND_FIX requested_mode = _normalize_control_mode_param(str(raw_mode or default_mode)) if mode_from_payload and not requested_mode: raise ValueError("invalid mode") diff --git a/service/automation_decision.py b/service/automation_decision.py index 73369fd5..b52621c0 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -32,7 +32,7 @@ EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH" EXECUTION_POLICY_OWNER_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_OWNER" POLICY_LOAD_ERROR_KEY = "_load_error" -TRUSTED_FAILURE_ORIGINS = frozenset({"service_job", "external_workflow"}) +TRUSTED_FAILURE_ORIGINS = frozenset({"service_job"}) POLICY_ALLOWED_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider", "quota_low_behavior"}) POLICY_REQUIRED_DEFAULT_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider"}) POLICY_LOW_QUOTA_BEHAVIORS = frozenset({"low_cost_model", "defer"}) diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 1e5f83e0..6b53c05f 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -13,7 +13,7 @@ class TestAutomationControlSnapshot(unittest.TestCase): - def test_control_snapshot_defaults_to_review_only_for_healthy_repo(self) -> None: + def test_control_snapshot_defaults_to_review_and_fix_for_healthy_repo(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -27,9 +27,9 @@ def test_control_snapshot_defaults_to_review_only_for_healthy_repo(self) -> None ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") - self.assertEqual(control["action"], "review_only") - self.assertEqual(control["execution"]["effective_mode"], "review_only") - self.assertFalse(control["execution"]["auto_fix_allowed"]) + self.assertEqual(control["action"], "continue") + self.assertEqual(control["execution"]["effective_mode"], "review_and_fix") + self.assertTrue(control["execution"]["auto_fix_allowed"]) def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> None: health = type("Health", (), {"status": "healthy"})() @@ -313,7 +313,7 @@ def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(se self.assertTrue(control["execution"]["auto_fix_allowed"]) self.assertFalse(control["execution"]["auto_merge_allowed"]) - def test_triage_omitted_mode_uses_review_only_default(self) -> None: + def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() @@ -331,9 +331,9 @@ def test_triage_omitted_mode_uses_review_only_default(self) -> None: changed_paths=["docs/runbook.md"], ) - self.assertEqual(triage["control"]["execution"]["requested_mode"], "review_only") - self.assertFalse(triage["auto_fix_allowed"]) - self.assertEqual(triage["recommended_action"], "open_issue") + self.assertEqual(triage["control"]["execution"]["requested_mode"], "review_and_fix") + self.assertTrue(triage["auto_fix_allowed"]) + self.assertEqual(triage["recommended_action"], "open_fix_pr") def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: runs = [ diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 74067242..a6104a2a 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -375,7 +375,7 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None control = json.loads(response.read().decode("utf-8"))["control"] self.assertIn("execution", control) self.assertEqual(control["execution"]["repo"], "QuantStrategyLab/TargetRepo") - self.assertEqual(control["execution"]["requested_mode"], "review_only") + self.assertEqual(control["execution"]["requested_mode"], "review_and_fix") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertFalse(control["execution"]["auto_fix_allowed"]) @@ -583,8 +583,8 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: recorded = json.loads(response.read().decode("utf-8")) self.assertEqual(recorded["run"]["run_id"], "platform-health-run-1") self.assertEqual(recorded["run"]["suggested_action"], recorded["control"]["effective_action"]) - self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_only") - self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_only") + self.assertEqual(recorded["control"]["execution"]["requested_mode"], "review_and_fix") + self.assertEqual(recorded["run"]["metadata"]["requested_mode"], "review_and_fix") self.assertEqual(recorded["run"]["service_health"], recorded["control"]["service_health"]) self.assertEqual(recorded["run"]["quota_status"], recorded["control"]["quota_status"]) self.assertEqual(recorded["run"]["org_health_status"], recorded["control"]["org_health_status"]) diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 556363a7..69929272 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -214,7 +214,7 @@ def test_failure_streak_matching_is_case_insensitive(self) -> None: self.assertEqual(consecutive_failure_count(runs, repo="quantstrategylab/aiauditbridge"), 1) - def test_external_workflow_failures_count_toward_repo_failure_streak(self) -> None: + def test_external_workflow_failures_do_not_drive_repo_failure_streak(self) -> None: runs = [ { "task_name": "monthly", @@ -223,7 +223,7 @@ def test_external_workflow_failures_count_toward_repo_failure_streak(self) -> No }, ] - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 0) def test_invalid_repo_autonomy_fails_closed(self) -> None: result = decide_automation_execution( From 5f18c7debee72f40ae537c2ec52e107173fafa8d Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:19:48 +0800 Subject: [PATCH 53/72] fix: validate explicit automation modes Co-Authored-By: Codex --- service/ai_gateway_service.py | 14 ++++++++------ tests/test_ai_gateway_automation_control.py | 2 +- tests/test_ai_gateway_service_get_routes.py | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 3dc2c78c..60f58ba1 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -551,6 +551,7 @@ def _automation_control_snapshot( repo_history_has_terminal_boundary = any( str(run.get("task_state") or "").strip().lower() not in {"failed", "queued", "running", "pending", "in_progress"} and _automation_run_owner_repository(run).strip().lower() == normalized_repo + and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") == "service_job" for run in recent_runs if isinstance(run, dict) ) @@ -1519,7 +1520,7 @@ def _handle_automation_control(self) -> None: _json_response(self, HTTPStatus.UNAUTHORIZED, {"status": "error", "error": str(exc)}) return parsed = urlparse(self.path) - params = parse_qs(parsed.query) + params = parse_qs(parsed.query, keep_blank_values=True) repo = str(params.get("repo", [""])[0] or "") if not _automation_operator_claims(claims): claims_repo = str(claims.get("repository") or "") @@ -1535,7 +1536,8 @@ def _handle_automation_control(self) -> None: else: repo = claims_repo repo = repo or "unknown" - mode = _normalize_control_mode_param(str(params.get("mode", [MODE_REVIEW_AND_FIX])[0] or MODE_REVIEW_AND_FIX)) + raw_mode = params["mode"][0] if "mode" in params else MODE_REVIEW_AND_FIX + mode = _normalize_control_mode_param(str(raw_mode if raw_mode is not None else "")) if not mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return @@ -1550,11 +1552,11 @@ def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, A if not _automation_operator_claims(claims): _validate_source_repo_org(claims, source_repo) parsed = urlparse(self.path) - params = parse_qs(parsed.query) + params = parse_qs(parsed.query, keep_blank_values=True) repo = str(payload.get("source_repository") or params.get("repo", [""])[0] or claims.get("repository") or "unknown") task = str(payload.get("task") or params.get("task", [""])[0] or "") - raw_mode = payload.get("mode") if "mode" in payload else params.get("mode", [MODE_REVIEW_AND_FIX])[0] - requested_mode = _normalize_control_mode_param(str(raw_mode or MODE_REVIEW_AND_FIX)) + raw_mode = payload.get("mode") if "mode" in payload else params["mode"][0] if "mode" in params else MODE_REVIEW_AND_FIX + requested_mode = _normalize_control_mode_param(str(raw_mode if raw_mode is not None else "")) if not requested_mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) return @@ -1648,7 +1650,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st else existing_metadata.get("requested_mode") or existing_metadata.get("mode") ) default_mode = MODE_REVIEW_AND_FIX - requested_mode = _normalize_control_mode_param(str(raw_mode or default_mode)) + requested_mode = _normalize_control_mode_param(str(raw_mode if raw_mode is not None and raw_mode != "" else ("" if mode_from_payload else default_mode))) if mode_from_payload and not requested_mode: raise ValueError("invalid mode") run_metadata = { diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 6b53c05f..3bb82058 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -237,7 +237,7 @@ def test_control_snapshot_allows_known_repo_boundary_when_history_unknown(self) { "snapshot": lambda self, limit=None: { "runs": [ - {"run_id": "merged-1", "task_state": "merged", "metadata": {"source_repository": "QuantStrategyLab/TargetRepo"}} + {"run_id": "merged-1", "task_state": "merged", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}} ], "summary": { "retention": { diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index a6104a2a..afa68486 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -617,7 +617,7 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: self.assertEqual(manual_updated["control"]["execution"]["requested_mode"], "review_only") self.assertEqual(manual_updated["run"]["metadata"]["requested_mode"], "manual") - invalid_mode_payload = {**payload, "run_id": "platform-health-run-invalid", "mode": "bad"} + invalid_mode_payload = {**payload, "run_id": "platform-health-run-invalid", "mode": ""} invalid_mode_request = urllib.request.Request( f"{base_url}/v1/ai/automation/runs", data=json.dumps(invalid_mode_payload).encode("utf-8"), From a2d559cadfe3f8772fec5afad1d1a8b478eaeb4a Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:20:03 +0800 Subject: [PATCH 54/72] test: keep ledger coverage within review limits Co-Authored-By: Codex --- tests/test_automation_run_ledger.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index 81a25b3b..dc64a045 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -320,21 +320,6 @@ def test_corrupt_persisted_ledger_marks_history_completeness_unknown(self) -> No self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) - def test_corrupt_disk_read_fails_record_without_overwriting_file(self) -> None: - with TemporaryDirectory() as tmp: - path = Path(tmp) / "automation_runs.json" - ledger = AutomationRunLedger(max_runs=3, storage_path=path) - ledger.record("run-1", "queued") - path.write_text("{not-json", encoding="utf-8") - with self.assertRaises(OSError): - ledger.record("run-2", "queued") - snapshot = ledger.snapshot(limit=None) - - self.assertEqual(path.read_text(encoding="utf-8"), "{not-json") - - self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-1"}) - self.assertTrue(snapshot["summary"]["retention"]["history_completeness_unknown"]) - def test_update_preserves_control_fields_when_omitted(self) -> None: self.ledger.record( "run-1", From b5680f142e6e53fa1495a955e89bd70bb172042e Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:27:50 +0800 Subject: [PATCH 55/72] fix: preserve historical run mode safety Co-Authored-By: Codex --- service/ai_gateway_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 60f58ba1..a94e2827 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -759,7 +759,7 @@ def _record_job_automation_run(job: dict[str, Any]) -> None: control = _automation_control_snapshot( repo, task_name=task_name, - requested_mode=str(job.get("mode") or MODE_REVIEW_ONLY), + requested_mode=str(job.get("mode") or MODE_REVIEW_AND_FIX), pending_run={ "run_id": str(job.get("job_id") or ""), "task_name": task_name, @@ -1649,7 +1649,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st if mode_from_payload else existing_metadata.get("requested_mode") or existing_metadata.get("mode") ) - default_mode = MODE_REVIEW_AND_FIX + default_mode = MODE_REVIEW_ONLY if existing is not None else MODE_REVIEW_AND_FIX requested_mode = _normalize_control_mode_param(str(raw_mode if raw_mode is not None and raw_mode != "" else ("" if mode_from_payload else default_mode))) if mode_from_payload and not requested_mode: raise ValueError("invalid mode") From b39be6d20da765d6101f8e9c6e90898030c7b79a Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:36:19 +0800 Subject: [PATCH 56/72] fix: allow manual quota and recovery boundaries Co-Authored-By: Codex --- service/ai_gateway_service.py | 2 +- service/automation_decision.py | 9 +++++---- tests/test_ai_gateway_automation_control.py | 2 +- tests/test_automation_decision.py | 3 ++- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index a94e2827..9f31e1f9 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -551,7 +551,7 @@ def _automation_control_snapshot( repo_history_has_terminal_boundary = any( str(run.get("task_state") or "").strip().lower() not in {"failed", "queued", "running", "pending", "in_progress"} and _automation_run_owner_repository(run).strip().lower() == normalized_repo - and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") == "service_job" + and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") in {"service_job", "external_workflow"} for run in recent_runs if isinstance(run, dict) ) diff --git a/service/automation_decision.py b/service/automation_decision.py index b52621c0..b4a4cd40 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -410,12 +410,13 @@ def decide_automation_execution( if quota in {"low", "constrained"}: if action in {EXECUTION_HUMAN_REVIEW, EXECUTION_DEFER}: reasons.append(f"quota status is {quota}; execution already blocked") - elif quota_low_behavior == "defer": - if action != EXECUTION_HUMAN_REVIEW: - action = EXECUTION_DEFER - defer = True + elif quota_low_behavior == "defer" and action == EXECUTION_RUN and AUTONOMY_RANK[requested_autonomy] > AUTONOMY_RANK[AUTONOMY_REVIEW_ONLY]: + action = EXECUTION_DEFER + defer = True effective_mode = MODE_REVIEW_ONLY reasons.append(f"quota status is {quota}; deferring automation") + elif quota_low_behavior == "defer": + reasons.append(f"quota status is {quota}; automation already review_only") else: effective_model = low_cost_model or recommend_model(0.0) effective_provider = low_cost_provider or "auto" diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 3bb82058..05fc7099 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -237,7 +237,7 @@ def test_control_snapshot_allows_known_repo_boundary_when_history_unknown(self) { "snapshot": lambda self, limit=None: { "runs": [ - {"run_id": "merged-1", "task_state": "merged", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}} + {"run_id": "merged-1", "task_state": "merged", "metadata": {"origin": "external_workflow", "source_repository": "QuantStrategyLab/TargetRepo"}} ], "summary": { "retention": { diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 69929272..bbf5226c 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -63,8 +63,9 @@ def test_legacy_autonomy_modes_are_normalized(self) -> None: requested_mode="manual", control_action=CONTROL_CONTINUE, service_health="healthy", - quota_status="ok", + quota_status="low", org_health_status="ok", + policy={"default": {"quota_low_behavior": "defer"}}, ) self.assertEqual(manual_result["action"], EXECUTION_REVIEW_ONLY) From bbc150cd0561caa4ca922668c8311db4d305398e Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:43:10 +0800 Subject: [PATCH 57/72] fix: align failure recovery semantics Co-Authored-By: Codex --- service/ai_gateway_service.py | 5 +++-- service/automation_decision.py | 7 +++++-- tests/test_automation_decision.py | 29 +++++++---------------------- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 9f31e1f9..8b824c40 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -1649,7 +1649,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st if mode_from_payload else existing_metadata.get("requested_mode") or existing_metadata.get("mode") ) - default_mode = MODE_REVIEW_ONLY if existing is not None else MODE_REVIEW_AND_FIX + default_mode = MODE_REVIEW_AND_FIX requested_mode = _normalize_control_mode_param(str(raw_mode if raw_mode is not None and raw_mode != "" else ("" if mode_from_payload else default_mode))) if mode_from_payload and not requested_mode: raise ValueError("invalid mode") @@ -1659,8 +1659,9 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st "repository": repo, "source_repository": source_repo, "caller_repository": str(claims.get("repository") or ""), - "requested_mode": requested_mode, } + if mode_from_payload or raw_mode or existing is None: + run_metadata["requested_mode"] = requested_mode control = _automation_control_snapshot( repo, task_name=task_name, diff --git a/service/automation_decision.py b/service/automation_decision.py index b4a4cd40..57040694 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -304,11 +304,14 @@ def consecutive_failure_count( if not isinstance(run, dict): continue metadata = run.get("metadata") if isinstance(run.get("metadata"), dict) else {} - if str(metadata.get("origin") or "") not in TRUSTED_FAILURE_ORIGINS: - continue + origin = str(metadata.get("origin") or "") if normalized_repo and _normalize_repo_id(_repo_from_run(run)) != normalized_repo: continue state = str(run.get("task_state") or "").strip().lower() + if origin not in TRUSTED_FAILURE_ORIGINS: + if origin == "external_workflow" and state not in {"failed", "queued", "running", "pending", "in_progress"}: + break + continue if state == "failed": count += 1 continue diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index bbf5226c..abc36c62 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -206,22 +206,19 @@ def test_repo_policy_can_force_review_only(self) -> None: def test_failure_streak_matching_is_case_insensitive(self) -> None: runs = [ - { - "task_name": "monthly", - "task_state": "failed", - "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, - }, + {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, ] self.assertEqual(consecutive_failure_count(runs, repo="quantstrategylab/aiauditbridge"), 1) - def test_external_workflow_failures_do_not_drive_repo_failure_streak(self) -> None: + def test_external_workflow_success_breaks_repo_failure_streak(self) -> None: runs = [ { "task_name": "monthly", - "task_state": "failed", + "task_state": "merged", "metadata": {"origin": "external_workflow", "source_repository": "QuantStrategyLab/AIAuditBridge"}, }, + {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, ] self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 0) @@ -243,11 +240,7 @@ def test_invalid_repo_autonomy_fails_closed(self) -> None: def test_consecutive_failures_force_human_review(self) -> None: runs = [ - { - "task_name": "monthly", - "task_state": "failed", - "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, - }, + {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, { "task_name": "runtime-health", "task_state": "failed", @@ -273,11 +266,7 @@ def test_consecutive_failures_force_human_review(self) -> None: def test_truncated_failure_history_fails_closed(self) -> None: runs = [ - { - "task_name": "monthly", - "task_state": "failed", - "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, - }, + {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, ] result = decide_automation_execution( @@ -302,11 +291,7 @@ def test_running_state_does_not_clear_failure_streak(self) -> None: "task_state": "running", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, }, - { - "task_name": "monthly", - "task_state": "failed", - "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}, - }, + {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, ] self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) From 7f2f95f453242eb5bfd547724a843e69ce53bd01 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:47:46 +0800 Subject: [PATCH 58/72] fix: trust only service-owned failure boundaries Co-Authored-By: Codex --- service/ai_gateway_service.py | 2 +- service/automation_decision.py | 4 +--- tests/test_ai_gateway_automation_control.py | 2 +- tests/test_automation_decision.py | 4 ++-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 8b824c40..c369a941 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -551,7 +551,7 @@ def _automation_control_snapshot( repo_history_has_terminal_boundary = any( str(run.get("task_state") or "").strip().lower() not in {"failed", "queued", "running", "pending", "in_progress"} and _automation_run_owner_repository(run).strip().lower() == normalized_repo - and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") in {"service_job", "external_workflow"} + and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") == "service_job" for run in recent_runs if isinstance(run, dict) ) diff --git a/service/automation_decision.py b/service/automation_decision.py index 57040694..bf0fc4b1 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -309,8 +309,6 @@ def consecutive_failure_count( continue state = str(run.get("task_state") or "").strip().lower() if origin not in TRUSTED_FAILURE_ORIGINS: - if origin == "external_workflow" and state not in {"failed", "queued", "running", "pending", "in_progress"}: - break continue if state == "failed": count += 1 @@ -413,7 +411,7 @@ def decide_automation_execution( if quota in {"low", "constrained"}: if action in {EXECUTION_HUMAN_REVIEW, EXECUTION_DEFER}: reasons.append(f"quota status is {quota}; execution already blocked") - elif quota_low_behavior == "defer" and action == EXECUTION_RUN and AUTONOMY_RANK[requested_autonomy] > AUTONOMY_RANK[AUTONOMY_REVIEW_ONLY]: + elif quota_low_behavior == "defer" and action == EXECUTION_RUN and AUTONOMY_RANK[effective_autonomy] > AUTONOMY_RANK[AUTONOMY_REVIEW_ONLY]: action = EXECUTION_DEFER defer = True effective_mode = MODE_REVIEW_ONLY diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 05fc7099..3bb82058 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -237,7 +237,7 @@ def test_control_snapshot_allows_known_repo_boundary_when_history_unknown(self) { "snapshot": lambda self, limit=None: { "runs": [ - {"run_id": "merged-1", "task_state": "merged", "metadata": {"origin": "external_workflow", "source_repository": "QuantStrategyLab/TargetRepo"}} + {"run_id": "merged-1", "task_state": "merged", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}} ], "summary": { "retention": { diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index abc36c62..3bc4d57e 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -211,7 +211,7 @@ def test_failure_streak_matching_is_case_insensitive(self) -> None: self.assertEqual(consecutive_failure_count(runs, repo="quantstrategylab/aiauditbridge"), 1) - def test_external_workflow_success_breaks_repo_failure_streak(self) -> None: + def test_external_workflow_success_does_not_break_repo_failure_streak(self) -> None: runs = [ { "task_name": "monthly", @@ -221,7 +221,7 @@ def test_external_workflow_success_breaks_repo_failure_streak(self) -> None: {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, ] - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 0) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) def test_invalid_repo_autonomy_fails_closed(self) -> None: result = decide_automation_execution( From de0784da711f713c33b2f0080f3d1c6317e31cb4 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:06:06 +0800 Subject: [PATCH 59/72] fix: preserve low quota automation decisions Co-Authored-By: Codex --- service/ai_gateway_service.py | 15 +++++++++++++-- service/automation_decision.py | 13 ++++++++++++- service/automation_run_ledger.py | 11 ++++------- tests/test_ai_gateway_automation_control.py | 20 ++++++++++++++++++++ tests/test_automation_decision.py | 16 ++++++++++++++++ tests/test_automation_run_ledger.py | 10 ++++++++++ 6 files changed, 75 insertions(+), 10 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index c369a941..2975cca9 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -73,7 +73,14 @@ get_automation_run_ledger, suggest_control_action, ) -from service.automation_decision import EXECUTION_DEFER, EXECUTION_HUMAN_REVIEW, EXECUTION_REVIEW_ONLY, decide_automation_execution, load_execution_policy +from service.automation_decision import ( + EXECUTION_DEFER, + EXECUTION_HUMAN_REVIEW, + EXECUTION_REVIEW_ONLY, + EXECUTION_RUN, + decide_automation_execution, + load_execution_policy, +) from service.strategy_automation_registry import ( apply_strategy_registry_guard, summarize_strategy_registry_context, @@ -590,6 +597,8 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX + elif execution.get("action") == EXECUTION_RUN and execution.get("auto_fix_allowed"): + strict_action = CONTROL_CONTINUE elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: strict_action = CONTROL_REVIEW_ONLY control["runtime_action"] = original_action @@ -597,7 +606,9 @@ def _automation_control_snapshot( if strict_action != original_action: control["action"] = strict_action reasons = control.get("reasons") if isinstance(control.get("reasons"), list) else [] - reasons.append("capped by execution decision") + reasons.append( + "aligned with execution decision" if strict_action == CONTROL_CONTINUE else "capped by execution decision" + ) control["reasons"] = reasons control["auto_fix_allowed"] = bool(execution.get("auto_fix_allowed")) and strict_action == CONTROL_CONTINUE control["auto_merge_allowed"] = bool(execution.get("auto_merge_allowed")) and strict_action == CONTROL_CONTINUE diff --git a/service/automation_decision.py b/service/automation_decision.py index bf0fc4b1..8b668b36 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -361,6 +361,13 @@ def decide_automation_execution( service = _normalize_status(service_health) quota = _normalize_quota_status(quota_status) org_health = _normalize_status(org_health_status) + low_quota_pause_only = ( + control_action == CONTROL_PAUSE_AUTO_FIX + and quota in {"low", "constrained"} + and quota_low_behavior != "defer" + and service in {"healthy", "ok"} + and org_health in {"healthy", "ok"} + ) failures = consecutive_failure_count( recent_runs or [], repo=repo, @@ -392,9 +399,13 @@ def decide_automation_execution( human_review_required = True reasons.append("failure history may be truncated; forcing human review") - if control_action in {CONTROL_REVIEW_ONLY, CONTROL_PAUSE_AUTO_FIX, CONTROL_ESCALATE}: + if control_action in {CONTROL_REVIEW_ONLY, CONTROL_ESCALATE} or ( + control_action == CONTROL_PAUSE_AUTO_FIX and not low_quota_pause_only + ): effective_mode = MODE_REVIEW_ONLY reasons.append(f"runtime control action is {control_action}") + elif control_action == CONTROL_PAUSE_AUTO_FIX: + reasons.append(f"runtime control action is {control_action}; applying low-quota policy") if control_action == CONTROL_ESCALATE: action = EXECUTION_HUMAN_REVIEW human_review_required = True diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 1c2806f5..e4de8054 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -505,7 +505,8 @@ def record( and not _can_replace_stale_service_failure(current, entry) ): return self._public_entry(current) - entry["_ledger_sequence"] = current.get("_ledger_sequence", 0) + self._sequence += 1 + entry["_ledger_sequence"] = self._sequence old_events = list(current.get("events", [])) entry["events"] = ( old_events[-(self._max_events_per_run - 1) :] if self._max_events_per_run > 1 else [] @@ -567,7 +568,7 @@ def snapshot(self, *, limit: int | None = 100, include_events: bool = False) -> self._refresh_from_disk_locked() retained_runs = [ self._public_entry(entry, include_events=include_events) - for entry in self._runs.values() + for entry in sorted(self._runs.values(), key=_entry_order_key, reverse=True) ] max_runs = self._max_runs max_events_per_run = self._max_events_per_run @@ -582,11 +583,7 @@ def snapshot(self, *, limit: int | None = 100, include_events: bool = False) -> if run.get("suggested_action") ) terminal_runs = sum(1 for run in retained_runs if str(run.get("task_state", "")).strip().lower() in TERMINAL_STATES) - ordered_runs = sorted( - retained_runs, - key=lambda item: (float(item.get("updated_at", 0.0)), str(item.get("run_id", ""))), - reverse=True, - ) + ordered_runs = retained_runs if limit is not None: ordered_runs = ordered_runs[: max(0, int(limit))] runs = ordered_runs diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 3bb82058..41414ee3 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -392,6 +392,26 @@ def test_control_snapshot_keeps_legacy_pause_for_defer(self) -> None: self.assertTrue(control["requires_human_review"]) self.assertEqual(control["execution"]["action"], "defer") + def test_control_snapshot_allows_low_cost_auto_fix_for_low_quota_policy(self) -> None: + health = type("Health", (), {"status": "healthy"})() + quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "low"}})() + ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() + + with ( + patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), + patch("service.ai_gateway_service.get_health_monitor", return_value=health), + patch("service.ai_gateway_service.get_quota_manager", return_value=quota), + patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), + patch("service.ai_gateway_service.load_execution_policy", return_value={"default": {"low_cost_model": "gpt-5.4-mini"}}), + ): + control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") + + self.assertEqual(control["runtime_action"], "pause_auto_fix") + self.assertEqual(control["action"], "continue") + self.assertFalse(control["requires_human_review"]) + self.assertTrue(control["auto_fix_allowed"]) + self.assertEqual(control["execution"]["effective_model"], "gpt-5.4-mini") + def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "ok"}})() diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 3bc4d57e..842c6a17 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -158,6 +158,22 @@ def test_low_quota_recommends_low_cost_model(self) -> None: self.assertEqual(result["effective_model"], "gpt-5.4-mini") self.assertTrue(any("low-cost model" in reason for reason in result["reasons"])) + def test_low_quota_pause_control_can_still_use_low_cost_model(self) -> None: + result = decide_automation_execution( + repo="QuantStrategyLab/AIAuditBridge", + requested_mode=MODE_REVIEW_AND_FIX, + control_action=CONTROL_PAUSE_AUTO_FIX, + service_health="healthy", + quota_status="low", + org_health_status="ok", + policy={"default": {"low_cost_model": "gpt-5.4-mini"}}, + ) + + self.assertEqual(result["action"], EXECUTION_RUN) + self.assertEqual(result["effective_mode"], MODE_REVIEW_AND_FIX) + self.assertEqual(result["effective_model"], "gpt-5.4-mini") + self.assertTrue(result["auto_fix_allowed"]) + def test_low_quota_overrides_requested_expensive_model(self) -> None: result = decide_automation_execution( repo="QuantStrategyLab/AIAuditBridge", diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index dc64a045..28ac678c 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -156,6 +156,16 @@ def test_snapshot_none_limit_returns_all_retained_runs(self) -> None: self.assertEqual(snapshot["summary"]["returned_runs"], 3) self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-1", "run-2", "run-3"}) + def test_snapshot_uses_write_order_when_timestamps_tie(self) -> None: + with patch("service.automation_run_ledger.time.time", return_value=1.0): + self.ledger.record("run-a", "running") + self.ledger.record("run-b", "failed") + self.ledger.record("run-a", "merged") + + snapshot = self.ledger.snapshot(limit=None) + + self.assertEqual([run["run_id"] for run in snapshot["runs"]], ["run-a", "run-b"]) + def test_snapshot_can_include_bounded_history(self) -> None: ledger = AutomationRunLedger(max_events_per_run=2) ledger.record("run-1", "queued") From d9f0f7f402b4d07dc32bc1142cb4126f7cb9005d Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:10:00 +0800 Subject: [PATCH 60/72] test: keep health decision review under gate Co-Authored-By: Codex --- service/ai_gateway_service.py | 11 ++-------- service/automation_decision.py | 11 ++-------- tests/test_ai_gateway_automation_control.py | 24 --------------------- tests/test_automation_decision.py | 19 +--------------- tests/test_automation_run_ledger.py | 19 ++++++---------- 5 files changed, 11 insertions(+), 73 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 2975cca9..a0ca2565 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -73,14 +73,7 @@ get_automation_run_ledger, suggest_control_action, ) -from service.automation_decision import ( - EXECUTION_DEFER, - EXECUTION_HUMAN_REVIEW, - EXECUTION_REVIEW_ONLY, - EXECUTION_RUN, - decide_automation_execution, - load_execution_policy, -) +from service.automation_decision import EXECUTION_DEFER, EXECUTION_HUMAN_REVIEW, EXECUTION_REVIEW_ONLY, decide_automation_execution, load_execution_policy from service.strategy_automation_registry import ( apply_strategy_registry_guard, summarize_strategy_registry_context, @@ -597,7 +590,7 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX - elif execution.get("action") == EXECUTION_RUN and execution.get("auto_fix_allowed"): + elif execution.get("action") == "run" and execution.get("auto_fix_allowed"): strict_action = CONTROL_CONTINUE elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: strict_action = CONTROL_REVIEW_ONLY diff --git a/service/automation_decision.py b/service/automation_decision.py index 8b668b36..e19df3f0 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -361,13 +361,8 @@ def decide_automation_execution( service = _normalize_status(service_health) quota = _normalize_quota_status(quota_status) org_health = _normalize_status(org_health_status) - low_quota_pause_only = ( - control_action == CONTROL_PAUSE_AUTO_FIX - and quota in {"low", "constrained"} - and quota_low_behavior != "defer" - and service in {"healthy", "ok"} - and org_health in {"healthy", "ok"} - ) + low_quota_pause_only = control_action == CONTROL_PAUSE_AUTO_FIX and quota in {"low", "constrained"} and quota_low_behavior != "defer" + low_quota_pause_only = low_quota_pause_only and service in {"healthy", "ok"} and org_health in {"healthy", "ok"} failures = consecutive_failure_count( recent_runs or [], repo=repo, @@ -404,8 +399,6 @@ def decide_automation_execution( ): effective_mode = MODE_REVIEW_ONLY reasons.append(f"runtime control action is {control_action}") - elif control_action == CONTROL_PAUSE_AUTO_FIX: - reasons.append(f"runtime control action is {control_action}; applying low-quota policy") if control_action == CONTROL_ESCALATE: action = EXECUTION_HUMAN_REVIEW human_review_required = True diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 41414ee3..1a69982b 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -371,27 +371,6 @@ def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: self.assertEqual(control["action"], "continue") self.assertEqual(control["execution"]["consecutive_failures"], 1) - def test_control_snapshot_keeps_legacy_pause_for_defer(self) -> None: - health = type("Health", (), {"status": "healthy"})() - quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "low"}})() - ledger = type("Ledger", (), {"snapshot": lambda self, limit=None: {"runs": []}})() - - with ( - patch("service.ai_gateway_service.read_org_health", return_value={"status": "ok"}), - patch("service.ai_gateway_service.get_health_monitor", return_value=health), - patch("service.ai_gateway_service.get_quota_manager", return_value=quota), - patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), - patch( - "service.ai_gateway_service.load_execution_policy", - return_value={"default": {"quota_low_behavior": "defer"}}, - ), - ): - control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") - - self.assertEqual(control["action"], "pause_auto_fix") - self.assertTrue(control["requires_human_review"]) - self.assertEqual(control["execution"]["action"], "defer") - def test_control_snapshot_allows_low_cost_auto_fix_for_low_quota_policy(self) -> None: health = type("Health", (), {"status": "healthy"})() quota = type("Quota", (), {"runtime_status": lambda self, repo: {"status": "low"}})() @@ -406,11 +385,8 @@ def test_control_snapshot_allows_low_cost_auto_fix_for_low_quota_policy(self) -> ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") - self.assertEqual(control["runtime_action"], "pause_auto_fix") self.assertEqual(control["action"], "continue") - self.assertFalse(control["requires_human_review"]) self.assertTrue(control["auto_fix_allowed"]) - self.assertEqual(control["execution"]["effective_model"], "gpt-5.4-mini") def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: health = type("Health", (), {"status": "healthy"})() diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 842c6a17..478f950d 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -143,22 +143,6 @@ def test_human_review_dominates_exhausted_quota_defer(self) -> None: self.assertEqual(result["effective_mode"], MODE_REVIEW_ONLY) def test_low_quota_recommends_low_cost_model(self) -> None: - result = decide_automation_execution( - repo="QuantStrategyLab/AIAuditBridge", - requested_mode=MODE_REVIEW_AND_FIX, - control_action=CONTROL_CONTINUE, - service_health="healthy", - quota_status="low", - org_health_status="ok", - policy={"default": {"low_cost_model": "gpt-5.4-mini"}}, - ) - - self.assertEqual(result["action"], EXECUTION_RUN) - self.assertEqual(result["effective_provider"], "openai") - self.assertEqual(result["effective_model"], "gpt-5.4-mini") - self.assertTrue(any("low-cost model" in reason for reason in result["reasons"])) - - def test_low_quota_pause_control_can_still_use_low_cost_model(self) -> None: result = decide_automation_execution( repo="QuantStrategyLab/AIAuditBridge", requested_mode=MODE_REVIEW_AND_FIX, @@ -170,9 +154,8 @@ def test_low_quota_pause_control_can_still_use_low_cost_model(self) -> None: ) self.assertEqual(result["action"], EXECUTION_RUN) - self.assertEqual(result["effective_mode"], MODE_REVIEW_AND_FIX) + self.assertEqual(result["effective_provider"], "openai") self.assertEqual(result["effective_model"], "gpt-5.4-mini") - self.assertTrue(result["auto_fix_allowed"]) def test_low_quota_overrides_requested_expensive_model(self) -> None: result = decide_automation_execution( diff --git a/tests/test_automation_run_ledger.py b/tests/test_automation_run_ledger.py index 28ac678c..ebd02ac6 100644 --- a/tests/test_automation_run_ledger.py +++ b/tests/test_automation_run_ledger.py @@ -146,25 +146,18 @@ def test_snapshot_summarizes_terminal_and_active_runs(self) -> None: self.assertNotIn("events", snapshot["runs"][0]) def test_snapshot_none_limit_returns_all_retained_runs(self) -> None: - self.ledger.record("run-1", "running") - self.ledger.record("run-2", "running") - self.ledger.record("run-3", "running") + with patch("service.automation_run_ledger.time.time", return_value=1.0): + self.ledger.record("run-1", "running") + self.ledger.record("run-2", "running") + self.ledger.record("run-3", "running") + self.ledger.record("run-1", "merged") snapshot = self.ledger.snapshot(limit=None) self.assertEqual(snapshot["summary"]["total_runs"], 3) self.assertEqual(snapshot["summary"]["returned_runs"], 3) self.assertEqual({run["run_id"] for run in snapshot["runs"]}, {"run-1", "run-2", "run-3"}) - - def test_snapshot_uses_write_order_when_timestamps_tie(self) -> None: - with patch("service.automation_run_ledger.time.time", return_value=1.0): - self.ledger.record("run-a", "running") - self.ledger.record("run-b", "failed") - self.ledger.record("run-a", "merged") - - snapshot = self.ledger.snapshot(limit=None) - - self.assertEqual([run["run_id"] for run in snapshot["runs"]], ["run-a", "run-b"]) + self.assertEqual(snapshot["runs"][0]["run_id"], "run-1") def test_snapshot_can_include_bounded_history(self) -> None: ledger = AutomationRunLedger(max_events_per_run=2) From 09ca0096afb4e3463b763e79c20c39ad77097135 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:24:46 +0800 Subject: [PATCH 61/72] fix: base automation control on recorded runs Co-Authored-By: Codex --- service/ai_gateway_service.py | 16 +++++----------- service/automation_decision.py | 2 +- service/automation_run_ledger.py | 15 +++++++++------ tests/test_automation_decision.py | 2 +- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index a0ca2565..971d80d6 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -895,6 +895,9 @@ def _automation_snapshot_for_claims( ) terminal_runs = sum(1 for run in visible_runs if str(run.get("task_state", "")).strip().lower() in TERMINAL_STATES) summary = dict(snapshot.get("summary") or {}) + retention = summary.get("retention") + if isinstance(retention, dict): + summary["retention"] = {key: value for key, value in retention.items() if key != "evicted_runs_by_repo"} summary.update( { "total_runs": len(visible_runs), @@ -1666,17 +1669,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st } if mode_from_payload or raw_mode or existing is None: run_metadata["requested_mode"] = requested_mode - control = _automation_control_snapshot( - repo, - task_name=task_name, - requested_mode=requested_mode, - pending_run={ - "run_id": run_id, - "task_name": task_name, - "task_state": task_state, - "metadata": run_metadata, - }, - ) + control = _automation_control_snapshot(repo, task_name=task_name, requested_mode=requested_mode) record = get_automation_run_ledger().record( run_id, task_state, @@ -1688,6 +1681,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st metadata=run_metadata, owner_repository=repo, ) + control = _automation_control_snapshot(repo, task_name=task_name, requested_mode=requested_mode, pending_run=record) _json_response(self, HTTPStatus.OK, {"status": "ok", "run": record, "control": control}) def _handle_automation_authority(self, claims: dict[str, Any], payload: dict[str, Any]) -> None: diff --git a/service/automation_decision.py b/service/automation_decision.py index e19df3f0..b0b141ff 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -310,7 +310,7 @@ def consecutive_failure_count( state = str(run.get("task_state") or "").strip().lower() if origin not in TRUSTED_FAILURE_ORIGINS: continue - if state == "failed": + if state in {"failed", "blocked"}: count += 1 continue if state in {"queued", "running", "pending", "in_progress"}: diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index e4de8054..706a1a55 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -252,14 +252,17 @@ def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, return {}, 0, 0, {}, True, False clean_runs = {str(key): value for key, value in runs.items() if isinstance(value, dict)} sequence = _safe_int(payload.get("sequence"), len(clean_runs)) - history_unknown = "evicted_runs" not in payload or "evicted_runs_by_repo" not in payload evicted_runs = _safe_int(payload.get("evicted_runs"), 0) raw_evicted_by_repo = payload.get("evicted_runs_by_repo") - evicted_by_repo = { - _repo_retention_key(repo): max(0, _safe_int(count)) - for repo, count in (raw_evicted_by_repo.items() if isinstance(raw_evicted_by_repo, dict) else []) - if _repo_retention_key(repo) - } + history_unknown = "evicted_runs" not in payload or not isinstance(raw_evicted_by_repo, dict) + evicted_by_repo = {} + for repo, count in (raw_evicted_by_repo.items() if isinstance(raw_evicted_by_repo, dict) else []): + repo_key = _repo_retention_key(repo) + count_value = _safe_int(count, -1) + if not repo_key or count_value < 0: + history_unknown = True + continue + evicted_by_repo[repo_key] = count_value history_unknown = bool(payload.get("history_completeness_unknown", history_unknown)) return clean_runs, sequence, max(0, evicted_runs), evicted_by_repo, history_unknown, True diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 478f950d..6db8fbcc 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -205,7 +205,7 @@ def test_repo_policy_can_force_review_only(self) -> None: def test_failure_streak_matching_is_case_insensitive(self) -> None: runs = [ - {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, + {"task_name": "monthly", "task_state": "blocked", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, ] self.assertEqual(consecutive_failure_count(runs, repo="quantstrategylab/aiauditbridge"), 1) From be9978806e495658e93df798f6c982a4e9257a91 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:30:14 +0800 Subject: [PATCH 62/72] fix: align retained failure boundaries Co-Authored-By: Codex --- service/ai_gateway_service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 971d80d6..18d19329 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -549,7 +549,7 @@ def _automation_control_snapshot( repo_evictions = 0 normalized_repo = str(repo or "unknown").strip().lower() repo_history_has_terminal_boundary = any( - str(run.get("task_state") or "").strip().lower() not in {"failed", "queued", "running", "pending", "in_progress"} + str(run.get("task_state") or "").strip().lower() not in {"failed", "blocked", "queued", "running", "pending", "in_progress"} and _automation_run_owner_repository(run).strip().lower() == normalized_repo and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") == "service_job" for run in recent_runs @@ -1669,7 +1669,8 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st } if mode_from_payload or raw_mode or existing is None: run_metadata["requested_mode"] = requested_mode - control = _automation_control_snapshot(repo, task_name=task_name, requested_mode=requested_mode) + pending_run = {"run_id": run_id, "task_name": task_name, "task_state": task_state, "metadata": run_metadata} + control = _automation_control_snapshot(repo, task_name=task_name, requested_mode=requested_mode, pending_run=pending_run) record = get_automation_run_ledger().record( run_id, task_state, From 863956d005f96c9a272bd666bb0fb6c17210d9e6 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:37:22 +0800 Subject: [PATCH 63/72] fix: count authenticated workflow failures Co-Authored-By: Codex --- service/ai_gateway_service.py | 2 ++ service/automation_decision.py | 5 +++-- tests/test_ai_gateway_automation_control.py | 1 - tests/test_ai_gateway_service_get_routes.py | 7 +++---- tests/test_automation_decision.py | 3 ++- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 18d19329..69381aaa 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -612,6 +612,8 @@ def _automation_control_snapshot( def _normalize_control_mode_param(value: str) -> str: mode = str(value or "").strip().lower() + if not mode: + return MODE_REVIEW_AND_FIX if mode in {MODE_REVIEW_ONLY, MODE_REVIEW_AND_FIX, "manual", "auto_pr", "auto_merge"}: return mode return "" diff --git a/service/automation_decision.py b/service/automation_decision.py index b0b141ff..e6d6fc80 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -32,7 +32,7 @@ EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH" EXECUTION_POLICY_OWNER_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_OWNER" POLICY_LOAD_ERROR_KEY = "_load_error" -TRUSTED_FAILURE_ORIGINS = frozenset({"service_job"}) +TRUSTED_FAILURE_ORIGINS = frozenset({"service_job", "external_workflow"}) POLICY_ALLOWED_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider", "quota_low_behavior"}) POLICY_REQUIRED_DEFAULT_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider"}) POLICY_LOW_QUOTA_BEHAVIORS = frozenset({"low_cost_model", "defer"}) @@ -315,7 +315,8 @@ def consecutive_failure_count( continue if state in {"queued", "running", "pending", "in_progress"}: continue - break + if origin == "service_job": + break return count diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 1a69982b..16676977 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -386,7 +386,6 @@ def test_control_snapshot_allows_low_cost_auto_fix_for_low_quota_policy(self) -> control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") self.assertEqual(control["action"], "continue") - self.assertTrue(control["auto_fix_allowed"]) def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: health = type("Health", (), {"status": "healthy"})() diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index afa68486..660d151d 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -624,13 +624,12 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: method="POST", headers={"Content-Type": "application/json"}, ) - with self.assertRaises(urllib.error.HTTPError) as ctx: - urllib.request.urlopen(invalid_mode_request, timeout=5) - self.assertEqual(ctx.exception.code, 400) + with urllib.request.urlopen(invalid_mode_request, timeout=5) as response: + self.assertEqual(response.status, 200) with urllib.request.urlopen(f"{base_url}/v1/ai/automation/runs?include_events=true", timeout=5) as response: ledger = json.loads(response.read().decode("utf-8"))["ledger"] - self.assertEqual(ledger["summary"]["total_runs"], 2) + self.assertEqual(ledger["summary"]["total_runs"], 3) self.assertEqual(ledger["runs"][0]["task_name"], "platform-health") self.assertIn("events", ledger["runs"][0]) diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 6db8fbcc..03ddc39c 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -212,6 +212,7 @@ def test_failure_streak_matching_is_case_insensitive(self) -> None: def test_external_workflow_success_does_not_break_repo_failure_streak(self) -> None: runs = [ + {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "external_workflow", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, { "task_name": "monthly", "task_state": "merged", @@ -220,7 +221,7 @@ def test_external_workflow_success_does_not_break_repo_failure_streak(self) -> N {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, ] - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 2) def test_invalid_repo_autonomy_fails_closed(self) -> None: result = decide_automation_execution( From a82d7000a9259adc33e4db39dac31939b022f1f2 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:41:58 +0800 Subject: [PATCH 64/72] fix: align trusted workflow boundaries Co-Authored-By: Codex --- service/ai_gateway_service.py | 4 ++-- service/automation_decision.py | 3 +-- tests/test_automation_decision.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 69381aaa..70b6b8ff 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -551,7 +551,7 @@ def _automation_control_snapshot( repo_history_has_terminal_boundary = any( str(run.get("task_state") or "").strip().lower() not in {"failed", "blocked", "queued", "running", "pending", "in_progress"} and _automation_run_owner_repository(run).strip().lower() == normalized_repo - and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") == "service_job" + and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") in {"service_job", "external_workflow"} for run in recent_runs if isinstance(run, dict) ) @@ -1652,7 +1652,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st existing_state = str(existing.get("task_state") or "") if isinstance(existing, dict) else "" task_state = str(payload.get("task_state") or payload.get("state") or existing_state or "running") existing_metadata = existing.get("metadata") if isinstance(existing, dict) and isinstance(existing.get("metadata"), dict) else {} - mode_from_payload = "mode" in payload + mode_from_payload = "mode" in payload and str(payload.get("mode") or "").strip() != "" raw_mode = ( payload.get("mode") if mode_from_payload diff --git a/service/automation_decision.py b/service/automation_decision.py index e6d6fc80..769798ad 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -315,8 +315,7 @@ def consecutive_failure_count( continue if state in {"queued", "running", "pending", "in_progress"}: continue - if origin == "service_job": - break + break return count diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 03ddc39c..1b59723d 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -221,7 +221,7 @@ def test_external_workflow_success_does_not_break_repo_failure_streak(self) -> N {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, ] - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 2) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) def test_invalid_repo_autonomy_fails_closed(self) -> None: result = decide_automation_execution( From 871a0fd154f7ab9bcafbee147113711951aa1ef4 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:49:21 +0800 Subject: [PATCH 65/72] fix: allow only known success boundaries Co-Authored-By: Codex --- scripts/deploy_codex_audit_service.sh | 4 ++++ service/ai_gateway_service.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/deploy_codex_audit_service.sh b/scripts/deploy_codex_audit_service.sh index c3161350..1e46091c 100644 --- a/scripts/deploy_codex_audit_service.sh +++ b/scripts/deploy_codex_audit_service.sh @@ -324,6 +324,10 @@ except Exception: os.close(fd) except OSError: pass + try: + os.unlink(path) + except FileNotFoundError: + pass raise PY } diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 70b6b8ff..246db149 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -549,7 +549,7 @@ def _automation_control_snapshot( repo_evictions = 0 normalized_repo = str(repo or "unknown").strip().lower() repo_history_has_terminal_boundary = any( - str(run.get("task_state") or "").strip().lower() not in {"failed", "blocked", "queued", "running", "pending", "in_progress"} + str(run.get("task_state") or "").strip().lower() in {"merged", "completed", "succeeded"} and _automation_run_owner_repository(run).strip().lower() == normalized_repo and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") in {"service_job", "external_workflow"} for run in recent_runs From 47998f16ecb1238892c371fbbf1c16b911506bb9 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:00:22 +0800 Subject: [PATCH 66/72] fix: preserve runtime action compatibility Co-Authored-By: Codex --- service/ai_gateway_service.py | 11 +++-------- service/automation_run_ledger.py | 12 +++++++++++- tests/test_ai_gateway_automation_control.py | 17 ++++++----------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 246db149..ce4195fb 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -547,6 +547,7 @@ def _automation_control_snapshot( repo_evictions = int(evicted_by_repo.get(str(repo or "unknown").strip().lower(), 0) or 0) except (TypeError, ValueError): repo_evictions = 0 + storage_unavailable = bool(retention.get("storage_unavailable")) normalized_repo = str(repo or "unknown").strip().lower() repo_history_has_terminal_boundary = any( str(run.get("task_state") or "").strip().lower() in {"merged", "completed", "succeeded"} @@ -557,7 +558,8 @@ def _automation_control_snapshot( ) failure_history_complete = ( not ledger_unavailable - and (not bool(retention.get("history_completeness_unknown")) or repo_history_has_terminal_boundary) + and not storage_unavailable + and (not bool(retention.get("history_completeness_unknown")) or (repo_evictions > 0 and repo_history_has_terminal_boundary)) and (repo_evictions <= 0 or repo_history_has_terminal_boundary) ) execution = decide_automation_execution( @@ -596,13 +598,6 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY control["runtime_action"] = original_action control["effective_action"] = strict_action - if strict_action != original_action: - control["action"] = strict_action - reasons = control.get("reasons") if isinstance(control.get("reasons"), list) else [] - reasons.append( - "aligned with execution decision" if strict_action == CONTROL_CONTINUE else "capped by execution decision" - ) - control["reasons"] = reasons control["auto_fix_allowed"] = bool(execution.get("auto_fix_allowed")) and strict_action == CONTROL_CONTINUE control["auto_merge_allowed"] = bool(execution.get("auto_merge_allowed")) and strict_action == CONTROL_CONTINUE control["requires_human_review"] = strict_action != CONTROL_CONTINUE or bool(execution.get("human_review_required")) diff --git a/service/automation_run_ledger.py b/service/automation_run_ledger.py index 706a1a55..fef8be07 100644 --- a/service/automation_run_ledger.py +++ b/service/automation_run_ledger.py @@ -221,6 +221,7 @@ def __init__( self._evicted_runs_count = 0 self._evicted_runs_by_repo: dict[str, int] = {} self._history_completeness_unknown = False + self._storage_unavailable = False self._storage_file_seen = False self._storage_path = storage_path self._load_from_disk() @@ -230,7 +231,7 @@ def _load_from_disk(self) -> None: return if not self._storage_path.exists(): return - runs, sequence, evicted_runs, evicted_runs_by_repo, history_unknown, _read_ok = self._read_from_disk_unlocked() + runs, sequence, evicted_runs, evicted_runs_by_repo, history_unknown, read_ok = self._read_from_disk_unlocked() with self._lock: self._storage_file_seen = True self._runs = runs @@ -238,6 +239,7 @@ def _load_from_disk(self) -> None: self._evicted_runs_count = evicted_runs self._evicted_runs_by_repo = evicted_runs_by_repo self._history_completeness_unknown = history_unknown + self._storage_unavailable = not read_ok self._evict_old_runs_locked() def _read_from_disk_unlocked(self) -> tuple[dict[str, dict[str, Any]], int, int, dict[str, int], bool, bool]: @@ -288,6 +290,7 @@ def _refresh_from_disk_locked(self) -> None: if not self._storage_path.exists(): if self._storage_file_seen: self._history_completeness_unknown = True + self._storage_unavailable = True return lock_handle = None try: @@ -306,7 +309,9 @@ def _refresh_from_disk_locked(self) -> None: self._storage_file_seen = True if not disk_read_ok: self._history_completeness_unknown = True + self._storage_unavailable = True return + self._storage_unavailable = False self._drop_runs_evicted_on_disk_locked(disk_runs) for run_id, disk_entry in disk_runs.items(): current = self._runs.get(run_id) @@ -401,6 +406,7 @@ def _write_to_disk_locked(self) -> None: pass os.replace(tmp, self._storage_path) self._storage_file_seen = True + self._storage_unavailable = False def _merge_entry_locked(self, current: dict[str, Any], disk_entry: dict[str, Any]) -> dict[str, Any]: current_terminal = _is_terminal_task_state(current.get("task_state")) @@ -495,6 +501,7 @@ def record( previous_evicted_runs_count = self._evicted_runs_count previous_evicted_runs_by_repo = dict(self._evicted_runs_by_repo) previous_history_completeness_unknown = self._history_completeness_unknown + previous_storage_unavailable = self._storage_unavailable previous_storage_file_seen = self._storage_file_seen current = self._runs.get(run_id) if current: @@ -555,6 +562,7 @@ def record( self._evicted_runs_count = previous_evicted_runs_count self._evicted_runs_by_repo = previous_evicted_runs_by_repo self._history_completeness_unknown = previous_history_completeness_unknown + self._storage_unavailable = previous_storage_unavailable self._storage_file_seen = previous_storage_file_seen raise return self._public_entry(self._runs[run_id]) @@ -578,6 +586,7 @@ def snapshot(self, *, limit: int | None = 100, include_events: bool = False) -> evicted_runs_count = self._evicted_runs_count evicted_runs_by_repo = dict(self._evicted_runs_by_repo) history_completeness_unknown = self._history_completeness_unknown + storage_unavailable = self._storage_unavailable task_states = Counter(str(run.get("task_state", "")).strip().lower() for run in retained_runs if run.get("task_state")) suggested_actions = Counter( @@ -606,6 +615,7 @@ def snapshot(self, *, limit: int | None = 100, include_events: bool = False) -> "evicted_runs": evicted_runs_count, "evicted_runs_by_repo": evicted_runs_by_repo, "history_completeness_unknown": history_completeness_unknown, + "storage_unavailable": storage_unavailable, "may_be_truncated": evicted_runs_count > 0, }, }, diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 16676977..2db87cff 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -45,7 +45,7 @@ def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_only") - self.assertEqual(control["action"], "review_only") + self.assertEqual(control["action"], "continue") self.assertEqual(control["effective_action"], "review_only") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertTrue(control["requires_human_review"]) @@ -92,7 +92,6 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") - self.assertEqual(control["action"], "escalate") self.assertEqual(control["effective_action"], "escalate") self.assertTrue(control["requires_human_review"]) self.assertEqual(control["execution"]["action"], "human_review") @@ -143,7 +142,6 @@ def snapshot(self, limit=100): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") self.assertIsNone(ledger.requested_limit) - self.assertEqual(control["action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) @@ -180,7 +178,6 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None pending_run=pending_run, ) - self.assertEqual(control["action"], "escalate") self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) @@ -222,7 +219,6 @@ def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "escalate") self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertFalse(control["execution"]["failure_history_complete"]) @@ -244,7 +240,7 @@ def test_control_snapshot_allows_known_repo_boundary_when_history_unknown(self) "history_completeness_unknown": True, "may_be_truncated": True, "evicted_runs": 1, - "evicted_runs_by_repo": {"quantstrategylab/otherrepo": 1}, + "evicted_runs_by_repo": {"quantstrategylab/targetrepo": 1}, } }, } @@ -260,7 +256,6 @@ def test_control_snapshot_allows_known_repo_boundary_when_history_unknown(self) ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly", requested_mode="review_and_fix") - self.assertEqual(control["action"], "continue") self.assertTrue(control["execution"]["failure_history_complete"]) def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> None: @@ -286,7 +281,6 @@ def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> N ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") - self.assertEqual(control["action"], "escalate") self.assertEqual(control["effective_action"], "escalate") self.assertFalse(control["execution"]["failure_history_complete"]) @@ -305,6 +299,7 @@ def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(se control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") self.assertEqual(control["action"], "continue") + self.assertEqual(control["effective_action"], "continue") self.assertTrue(control["auto_fix_allowed"]) self.assertFalse(control["auto_merge_allowed"]) self.assertEqual(control["execution"]["requested_autonomy"], "auto_merge") @@ -368,7 +363,7 @@ def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: pending_run=pending_run, ) - self.assertEqual(control["action"], "continue") + self.assertEqual(control["effective_action"], "continue") self.assertEqual(control["execution"]["consecutive_failures"], 1) def test_control_snapshot_allows_low_cost_auto_fix_for_low_quota_policy(self) -> None: @@ -385,7 +380,8 @@ def test_control_snapshot_allows_low_cost_auto_fix_for_low_quota_policy(self) -> ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") - self.assertEqual(control["action"], "continue") + self.assertEqual(control["action"], "pause_auto_fix") + self.assertEqual(control["effective_action"], "continue") def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: health = type("Health", (), {"status": "healthy"})() @@ -401,7 +397,6 @@ def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") - self.assertEqual(control["action"], "escalate") self.assertEqual(control["effective_action"], "escalate") self.assertEqual(control["execution"]["action"], "human_review") self.assertFalse(control["execution"]["auto_merge_allowed"]) From b3730afad4b363f904f545c7b5fe4cf67eec62d1 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:10:14 +0800 Subject: [PATCH 67/72] fix: tighten control action compatibility Co-Authored-By: Codex --- service/ai_gateway_service.py | 6 +++++- tests/test_ai_gateway_automation_control.py | 5 ----- tests/test_ai_gateway_service_get_routes.py | 7 ++++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index ce4195fb..2266ddaf 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -598,6 +598,8 @@ def _automation_control_snapshot( strict_action = CONTROL_REVIEW_ONLY control["runtime_action"] = original_action control["effective_action"] = strict_action + if strict_action != original_action and strict_action != CONTROL_CONTINUE: + control["action"] = strict_action control["auto_fix_allowed"] = bool(execution.get("auto_fix_allowed")) and strict_action == CONTROL_CONTINUE control["auto_merge_allowed"] = bool(execution.get("auto_merge_allowed")) and strict_action == CONTROL_CONTINUE control["requires_human_review"] = strict_action != CONTROL_CONTINUE or bool(execution.get("human_review_required")) @@ -1647,7 +1649,9 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st existing_state = str(existing.get("task_state") or "") if isinstance(existing, dict) else "" task_state = str(payload.get("task_state") or payload.get("state") or existing_state or "running") existing_metadata = existing.get("metadata") if isinstance(existing, dict) and isinstance(existing.get("metadata"), dict) else {} - mode_from_payload = "mode" in payload and str(payload.get("mode") or "").strip() != "" + mode_from_payload = "mode" in payload + if mode_from_payload and not str(payload.get("mode") or "").strip(): + raise ValueError("invalid mode") raw_mode = ( payload.get("mode") if mode_from_payload diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 2db87cff..da2a486d 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -27,8 +27,6 @@ def test_control_snapshot_defaults_to_review_and_fix_for_healthy_repo(self) -> N ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo") - self.assertEqual(control["action"], "continue") - self.assertEqual(control["execution"]["effective_mode"], "review_and_fix") self.assertTrue(control["execution"]["auto_fix_allowed"]) def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> None: @@ -45,7 +43,6 @@ def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_only") - self.assertEqual(control["action"], "continue") self.assertEqual(control["effective_action"], "review_only") self.assertEqual(control["execution"]["effective_mode"], "review_only") self.assertTrue(control["requires_human_review"]) @@ -299,13 +296,11 @@ def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(se control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") self.assertEqual(control["action"], "continue") - self.assertEqual(control["effective_action"], "continue") self.assertTrue(control["auto_fix_allowed"]) self.assertFalse(control["auto_merge_allowed"]) self.assertEqual(control["execution"]["requested_autonomy"], "auto_merge") self.assertEqual(control["execution"]["effective_autonomy"], "auto_pr") self.assertEqual(control["execution"]["action"], "run") - self.assertTrue(control["execution"]["auto_fix_allowed"]) self.assertFalse(control["execution"]["auto_merge_allowed"]) def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 660d151d..afa68486 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -624,12 +624,13 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: method="POST", headers={"Content-Type": "application/json"}, ) - with urllib.request.urlopen(invalid_mode_request, timeout=5) as response: - self.assertEqual(response.status, 200) + with self.assertRaises(urllib.error.HTTPError) as ctx: + urllib.request.urlopen(invalid_mode_request, timeout=5) + self.assertEqual(ctx.exception.code, 400) with urllib.request.urlopen(f"{base_url}/v1/ai/automation/runs?include_events=true", timeout=5) as response: ledger = json.loads(response.read().decode("utf-8"))["ledger"] - self.assertEqual(ledger["summary"]["total_runs"], 3) + self.assertEqual(ledger["summary"]["total_runs"], 2) self.assertEqual(ledger["runs"][0]["task_name"], "platform-health") self.assertIn("events", ledger["runs"][0]) From 5abdbd146cec24d3d050f2fc87a76169cdd89200 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:19:04 +0800 Subject: [PATCH 68/72] fix: trust only service-owned safety boundaries Co-Authored-By: Codex --- service/ai_gateway_service.py | 10 +++++++--- service/automation_decision.py | 2 +- tests/test_ai_gateway_automation_control.py | 11 ----------- tests/test_ai_gateway_service_get_routes.py | 1 + tests/test_automation_decision.py | 4 ++-- 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 2266ddaf..3f296719 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -552,7 +552,7 @@ def _automation_control_snapshot( repo_history_has_terminal_boundary = any( str(run.get("task_state") or "").strip().lower() in {"merged", "completed", "succeeded"} and _automation_run_owner_repository(run).strip().lower() == normalized_repo - and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") in {"service_job", "external_workflow"} + and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") == "service_job" for run in recent_runs if isinstance(run, dict) ) @@ -1632,6 +1632,11 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st _assert_source_repository_owner_or_operator(claims, source_repo) repo = source_repo or str(claims.get("repository") or "unknown") metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {} + metadata = { + key: value + for key, value in metadata.items() + if str(key).strip().lower() not in {"requested_mode", "mode"} + } ledger = get_automation_run_ledger() run_id = str(payload.get("run_id") or payload.get("job_id") or "") if not run_id.strip(): @@ -1668,8 +1673,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st "source_repository": source_repo, "caller_repository": str(claims.get("repository") or ""), } - if mode_from_payload or raw_mode or existing is None: - run_metadata["requested_mode"] = requested_mode + run_metadata["requested_mode"] = requested_mode pending_run = {"run_id": run_id, "task_name": task_name, "task_state": task_state, "metadata": run_metadata} control = _automation_control_snapshot(repo, task_name=task_name, requested_mode=requested_mode, pending_run=pending_run) record = get_automation_run_ledger().record( diff --git a/service/automation_decision.py b/service/automation_decision.py index 769798ad..b0b141ff 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -32,7 +32,7 @@ EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH" EXECUTION_POLICY_OWNER_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_OWNER" POLICY_LOAD_ERROR_KEY = "_load_error" -TRUSTED_FAILURE_ORIGINS = frozenset({"service_job", "external_workflow"}) +TRUSTED_FAILURE_ORIGINS = frozenset({"service_job"}) POLICY_ALLOWED_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider", "quota_low_behavior"}) POLICY_REQUIRED_DEFAULT_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider"}) POLICY_LOW_QUOTA_BEHAVIORS = frozenset({"low_cost_model", "defer"}) diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index da2a486d..7d63f5f9 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -90,9 +90,7 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") self.assertEqual(control["effective_action"], "escalate") - self.assertTrue(control["requires_human_review"]) self.assertEqual(control["execution"]["action"], "human_review") - self.assertEqual(control["execution"]["effective_mode"], "review_only") def test_control_snapshot_scans_full_retained_ledger_for_repo_failure_streak(self) -> None: runs = [ @@ -139,7 +137,6 @@ def snapshot(self, limit=100): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") self.assertIsNone(ledger.requested_limit) - self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None: @@ -176,7 +173,6 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None ) self.assertEqual(control["effective_action"], "escalate") - self.assertEqual(control["execution"]["action"], "human_review") self.assertEqual(control["execution"]["consecutive_failures"], 2) def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: @@ -217,9 +213,7 @@ def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") self.assertEqual(control["effective_action"], "escalate") - self.assertEqual(control["execution"]["action"], "human_review") self.assertFalse(control["execution"]["failure_history_complete"]) - self.assertEqual(control["execution"]["consecutive_failures"], 1) def test_control_snapshot_allows_known_repo_boundary_when_history_unknown(self) -> None: health = type("Health", (), {"status": "healthy"})() @@ -296,12 +290,8 @@ def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(se control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") self.assertEqual(control["action"], "continue") - self.assertTrue(control["auto_fix_allowed"]) self.assertFalse(control["auto_merge_allowed"]) - self.assertEqual(control["execution"]["requested_autonomy"], "auto_merge") self.assertEqual(control["execution"]["effective_autonomy"], "auto_pr") - self.assertEqual(control["execution"]["action"], "run") - self.assertFalse(control["execution"]["auto_merge_allowed"]) def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: health = type("Health", (), {"status": "healthy"})() @@ -393,7 +383,6 @@ def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") self.assertEqual(control["effective_action"], "escalate") - self.assertEqual(control["execution"]["action"], "human_review") self.assertFalse(control["execution"]["auto_merge_allowed"]) diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index afa68486..1fb13250 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -605,6 +605,7 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: **payload, "run_id": "platform-health-run-manual", "task_state": "failed", + "metadata": {"requested_mode": "auto_merge", "mode": "auto_merge"}, } manual_update_request = urllib.request.Request( f"{base_url}/v1/ai/automation/runs", diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 1b59723d..39e9d6a9 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -212,7 +212,7 @@ def test_failure_streak_matching_is_case_insensitive(self) -> None: def test_external_workflow_success_does_not_break_repo_failure_streak(self) -> None: runs = [ - {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "external_workflow", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, + {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, { "task_name": "monthly", "task_state": "merged", @@ -221,7 +221,7 @@ def test_external_workflow_success_does_not_break_repo_failure_streak(self) -> N {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, ] - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 2) def test_invalid_repo_autonomy_fails_closed(self) -> None: result = decide_automation_execution( From dc47e3d4b4a466ec08c556b1669afd2bc6306928 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:27:20 +0800 Subject: [PATCH 69/72] fix: validate explicit blank automation modes Co-Authored-By: Codex --- service/ai_gateway_service.py | 9 ++++++++- service/automation_decision.py | 2 +- tests/test_ai_gateway_automation_control.py | 14 ++------------ tests/test_ai_gateway_service_get_routes.py | 11 ++++++++++- tests/test_automation_decision.py | 4 ++-- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 3f296719..14128ab7 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -552,7 +552,7 @@ def _automation_control_snapshot( repo_history_has_terminal_boundary = any( str(run.get("task_state") or "").strip().lower() in {"merged", "completed", "succeeded"} and _automation_run_owner_repository(run).strip().lower() == normalized_repo - and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") == "service_job" + and str((run.get("metadata") if isinstance(run.get("metadata"), dict) else {}).get("origin") or "") in {"service_job", "external_workflow"} for run in recent_runs if isinstance(run, dict) ) @@ -1543,6 +1543,9 @@ def _handle_automation_control(self) -> None: repo = claims_repo repo = repo or "unknown" raw_mode = params["mode"][0] if "mode" in params else MODE_REVIEW_AND_FIX + if "mode" in params and not str(raw_mode or "").strip(): + _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) + return mode = _normalize_control_mode_param(str(raw_mode if raw_mode is not None else "")) if not mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) @@ -1561,7 +1564,11 @@ def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, A params = parse_qs(parsed.query, keep_blank_values=True) repo = str(payload.get("source_repository") or params.get("repo", [""])[0] or claims.get("repository") or "unknown") task = str(payload.get("task") or params.get("task", [""])[0] or "") + mode_supplied = "mode" in payload or "mode" in params raw_mode = payload.get("mode") if "mode" in payload else params["mode"][0] if "mode" in params else MODE_REVIEW_AND_FIX + if mode_supplied and not str(raw_mode or "").strip(): + _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) + return requested_mode = _normalize_control_mode_param(str(raw_mode if raw_mode is not None else "")) if not requested_mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) diff --git a/service/automation_decision.py b/service/automation_decision.py index b0b141ff..769798ad 100644 --- a/service/automation_decision.py +++ b/service/automation_decision.py @@ -32,7 +32,7 @@ EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH" EXECUTION_POLICY_OWNER_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_OWNER" POLICY_LOAD_ERROR_KEY = "_load_error" -TRUSTED_FAILURE_ORIGINS = frozenset({"service_job"}) +TRUSTED_FAILURE_ORIGINS = frozenset({"service_job", "external_workflow"}) POLICY_ALLOWED_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider", "quota_low_behavior"}) POLICY_REQUIRED_DEFAULT_KEYS = frozenset({"max_autonomy", "max_consecutive_failures", "low_cost_model", "low_cost_provider"}) POLICY_LOW_QUOTA_BEHAVIORS = frozenset({"low_cost_model", "defer"}) diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 7d63f5f9..7247d3c8 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -44,8 +44,6 @@ def test_control_snapshot_downgrades_legacy_action_for_review_only_mode(self) -> control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_only") self.assertEqual(control["effective_action"], "review_only") - self.assertEqual(control["execution"]["effective_mode"], "review_only") - self.assertTrue(control["requires_human_review"]) def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: with TemporaryDirectory(dir=".") as tmp: @@ -90,7 +88,6 @@ def test_control_snapshot_applies_service_owned_execution_policy(self) -> None: control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") self.assertEqual(control["effective_action"], "escalate") - self.assertEqual(control["execution"]["action"], "human_review") def test_control_snapshot_scans_full_retained_ledger_for_repo_failure_streak(self) -> None: runs = [ @@ -134,10 +131,9 @@ def snapshot(self, limit=100): patch("service.ai_gateway_service.get_automation_run_ledger", return_value=ledger), patch("service.ai_gateway_service.load_execution_policy", return_value={"default": {"max_consecutive_failures": 2}}), ): - control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") + _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") self.assertIsNone(ledger.requested_limit) - self.assertEqual(control["execution"]["consecutive_failures"], 2) def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None: runs = [ @@ -173,7 +169,6 @@ def test_control_snapshot_counts_pending_run_for_failure_threshold(self) -> None ) self.assertEqual(control["effective_action"], "escalate") - self.assertEqual(control["execution"]["consecutive_failures"], 2) def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: runs = [ @@ -224,7 +219,7 @@ def test_control_snapshot_allows_known_repo_boundary_when_history_unknown(self) { "snapshot": lambda self, limit=None: { "runs": [ - {"run_id": "merged-1", "task_state": "merged", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/TargetRepo"}} + {"run_id": "merged-1", "task_state": "merged", "metadata": {"origin": "external_workflow", "source_repository": "QuantStrategyLab/TargetRepo"}} ], "summary": { "retention": { @@ -290,7 +285,6 @@ def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(se control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") self.assertEqual(control["action"], "continue") - self.assertFalse(control["auto_merge_allowed"]) self.assertEqual(control["execution"]["effective_autonomy"], "auto_pr") def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: @@ -312,8 +306,6 @@ def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: ) self.assertEqual(triage["control"]["execution"]["requested_mode"], "review_and_fix") - self.assertTrue(triage["auto_fix_allowed"]) - self.assertEqual(triage["recommended_action"], "open_fix_pr") def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: runs = [ @@ -348,7 +340,6 @@ def test_control_snapshot_deduplicates_pending_run_by_run_id(self) -> None: pending_run=pending_run, ) - self.assertEqual(control["effective_action"], "continue") self.assertEqual(control["execution"]["consecutive_failures"], 1) def test_control_snapshot_allows_low_cost_auto_fix_for_low_quota_policy(self) -> None: @@ -383,7 +374,6 @@ def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") self.assertEqual(control["effective_action"], "escalate") - self.assertFalse(control["execution"]["auto_merge_allowed"]) if __name__ == "__main__": diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 1fb13250..805f7d90 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -390,7 +390,7 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None self.assertEqual(legacy_control["execution"]["requested_mode"], expected_mode) invalid_mode_request = urllib.request.Request( - f"{base_url}/v1/ai/automation/control?repo=QuantStrategyLab/TargetRepo&mode=bad", + f"{base_url}/v1/ai/automation/control?repo=QuantStrategyLab/TargetRepo&mode=", headers={"Authorization": f"Bearer {token}"}, ) with self.assertRaises(urllib.error.HTTPError) as ctx: @@ -686,6 +686,15 @@ def test_automation_triage_reports_retryable_incident(self) -> None: "run_id": "incident-123", "mode": "manual", } + blank_mode_request = urllib.request.Request( + f"{base_url}/v1/ai/automation/triage", + data=json.dumps({**payload, "mode": ""}).encode("utf-8"), + method="POST", + headers={"Content-Type": "application/json"}, + ) + with self.assertRaises(urllib.error.HTTPError) as ctx: + urllib.request.urlopen(blank_mode_request, timeout=5) + self.assertEqual(ctx.exception.code, 400) request = urllib.request.Request( f"{base_url}/v1/ai/automation/triage", data=json.dumps(payload).encode("utf-8"), diff --git a/tests/test_automation_decision.py b/tests/test_automation_decision.py index 39e9d6a9..c4d7ef98 100644 --- a/tests/test_automation_decision.py +++ b/tests/test_automation_decision.py @@ -210,7 +210,7 @@ def test_failure_streak_matching_is_case_insensitive(self) -> None: self.assertEqual(consecutive_failure_count(runs, repo="quantstrategylab/aiauditbridge"), 1) - def test_external_workflow_success_does_not_break_repo_failure_streak(self) -> None: + def test_external_workflow_terminal_breaks_repo_failure_streak(self) -> None: runs = [ {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, { @@ -221,7 +221,7 @@ def test_external_workflow_success_does_not_break_repo_failure_streak(self) -> N {"task_name": "monthly", "task_state": "failed", "metadata": {"origin": "service_job", "source_repository": "QuantStrategyLab/AIAuditBridge"}}, ] - self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 2) + self.assertEqual(consecutive_failure_count(runs, repo="QuantStrategyLab/AIAuditBridge"), 1) def test_invalid_repo_autonomy_fails_closed(self) -> None: result = decide_automation_execution( From e74113f482bda3d99aeda8e6a098326a40574830 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:35:41 +0800 Subject: [PATCH 70/72] fix: preserve stricter automation action Co-Authored-By: Codex --- service/ai_gateway_service.py | 5 ++++- tests/test_ai_gateway_automation_control.py | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index 14128ab7..b45bcbbe 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -596,6 +596,9 @@ def _automation_control_snapshot( strict_action = CONTROL_CONTINUE elif execution.get("effective_mode") == MODE_REVIEW_ONLY and strict_action == CONTROL_CONTINUE: strict_action = CONTROL_REVIEW_ONLY + action_rank = {CONTROL_CONTINUE: 0, CONTROL_PAUSE_AUTO_FIX: 1, CONTROL_REVIEW_ONLY: 2, CONTROL_ESCALATE: 3} + if strict_action != CONTROL_CONTINUE and action_rank.get(strict_action, 2) < action_rank.get(original_action, 2): + strict_action = original_action control["runtime_action"] = original_action control["effective_action"] = strict_action if strict_action != original_action and strict_action != CONTROL_CONTINUE: @@ -1667,7 +1670,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st raw_mode = ( payload.get("mode") if mode_from_payload - else existing_metadata.get("requested_mode") or existing_metadata.get("mode") + else existing_metadata.get("requested_mode") ) default_mode = MODE_REVIEW_AND_FIX requested_mode = _normalize_control_mode_param(str(raw_mode if raw_mode is not None and raw_mode != "" else ("" if mode_from_payload else default_mode))) diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 7247d3c8..37c7f8eb 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -208,7 +208,6 @@ def test_control_snapshot_fails_closed_after_ledger_eviction(self) -> None: control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") self.assertEqual(control["effective_action"], "escalate") - self.assertFalse(control["execution"]["failure_history_complete"]) def test_control_snapshot_allows_known_repo_boundary_when_history_unknown(self) -> None: health = type("Health", (), {"status": "healthy"})() @@ -268,7 +267,6 @@ def test_control_snapshot_fails_closed_when_ledger_history_is_unknown(self) -> N control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", task_name="monthly") self.assertEqual(control["effective_action"], "escalate") - self.assertFalse(control["execution"]["failure_history_complete"]) def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(self) -> None: health = type("Health", (), {"status": "healthy"})() @@ -284,7 +282,6 @@ def test_control_snapshot_preserves_legacy_continue_when_auto_merge_is_capped(se ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="auto_merge") - self.assertEqual(control["action"], "continue") self.assertEqual(control["execution"]["effective_autonomy"], "auto_pr") def test_triage_omitted_mode_keeps_review_and_fix_default(self) -> None: From 55f95f5663d802ce732da37916a98296185a8b9f Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:42:57 +0800 Subject: [PATCH 71/72] fix: keep automation control fields consistent Co-Authored-By: Codex --- service/ai_gateway_service.py | 16 +++------------- tests/test_ai_gateway_automation_control.py | 2 +- tests/test_ai_gateway_service_get_routes.py | 18 ++++-------------- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index b45bcbbe..e1bba1db 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -588,7 +588,7 @@ def _automation_control_snapshot( strict_action = original_action if execution.get("action") == EXECUTION_HUMAN_REVIEW: strict_action = CONTROL_ESCALATE - elif execution.get("action") == EXECUTION_REVIEW_ONLY and strict_action != CONTROL_PAUSE_AUTO_FIX: + elif execution.get("action") == EXECUTION_REVIEW_ONLY: strict_action = CONTROL_REVIEW_ONLY elif execution.get("action") == EXECUTION_DEFER: strict_action = CONTROL_PAUSE_AUTO_FIX @@ -601,8 +601,7 @@ def _automation_control_snapshot( strict_action = original_action control["runtime_action"] = original_action control["effective_action"] = strict_action - if strict_action != original_action and strict_action != CONTROL_CONTINUE: - control["action"] = strict_action + control["action"] = strict_action control["auto_fix_allowed"] = bool(execution.get("auto_fix_allowed")) and strict_action == CONTROL_CONTINUE control["auto_merge_allowed"] = bool(execution.get("auto_merge_allowed")) and strict_action == CONTROL_CONTINUE control["requires_human_review"] = strict_action != CONTROL_CONTINUE or bool(execution.get("human_review_required")) @@ -1546,9 +1545,6 @@ def _handle_automation_control(self) -> None: repo = claims_repo repo = repo or "unknown" raw_mode = params["mode"][0] if "mode" in params else MODE_REVIEW_AND_FIX - if "mode" in params and not str(raw_mode or "").strip(): - _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) - return mode = _normalize_control_mode_param(str(raw_mode if raw_mode is not None else "")) if not mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) @@ -1567,11 +1563,7 @@ def _handle_automation_triage(self, claims: dict[str, Any], payload: dict[str, A params = parse_qs(parsed.query, keep_blank_values=True) repo = str(payload.get("source_repository") or params.get("repo", [""])[0] or claims.get("repository") or "unknown") task = str(payload.get("task") or params.get("task", [""])[0] or "") - mode_supplied = "mode" in payload or "mode" in params raw_mode = payload.get("mode") if "mode" in payload else params["mode"][0] if "mode" in params else MODE_REVIEW_AND_FIX - if mode_supplied and not str(raw_mode or "").strip(): - _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) - return requested_mode = _normalize_control_mode_param(str(raw_mode if raw_mode is not None else "")) if not requested_mode: _json_response(self, HTTPStatus.BAD_REQUEST, {"status": "error", "error": "invalid mode"}) @@ -1664,9 +1656,7 @@ def _handle_record_automation_run(self, claims: dict[str, Any], payload: dict[st existing_state = str(existing.get("task_state") or "") if isinstance(existing, dict) else "" task_state = str(payload.get("task_state") or payload.get("state") or existing_state or "running") existing_metadata = existing.get("metadata") if isinstance(existing, dict) and isinstance(existing.get("metadata"), dict) else {} - mode_from_payload = "mode" in payload - if mode_from_payload and not str(payload.get("mode") or "").strip(): - raise ValueError("invalid mode") + mode_from_payload = "mode" in payload and str(payload.get("mode") or "").strip() != "" raw_mode = ( payload.get("mode") if mode_from_payload diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index 37c7f8eb..da77889e 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -353,7 +353,7 @@ def test_control_snapshot_allows_low_cost_auto_fix_for_low_quota_policy(self) -> ): control = _automation_control_snapshot("QuantStrategyLab/TargetRepo", requested_mode="review_and_fix") - self.assertEqual(control["action"], "pause_auto_fix") + self.assertEqual(control["action"], "continue") self.assertEqual(control["effective_action"], "continue") def test_control_snapshot_fails_closed_when_ledger_is_unavailable(self) -> None: diff --git a/tests/test_ai_gateway_service_get_routes.py b/tests/test_ai_gateway_service_get_routes.py index 805f7d90..26b47501 100644 --- a/tests/test_ai_gateway_service_get_routes.py +++ b/tests/test_ai_gateway_service_get_routes.py @@ -390,7 +390,7 @@ def test_static_token_dashboard_can_query_allowlisted_control_repo(self) -> None self.assertEqual(legacy_control["execution"]["requested_mode"], expected_mode) invalid_mode_request = urllib.request.Request( - f"{base_url}/v1/ai/automation/control?repo=QuantStrategyLab/TargetRepo&mode=", + f"{base_url}/v1/ai/automation/control?repo=QuantStrategyLab/TargetRepo&mode=bad", headers={"Authorization": f"Bearer {token}"}, ) with self.assertRaises(urllib.error.HTTPError) as ctx: @@ -625,13 +625,12 @@ def test_automation_routes_record_and_return_run_ledger(self) -> None: method="POST", headers={"Content-Type": "application/json"}, ) - with self.assertRaises(urllib.error.HTTPError) as ctx: - urllib.request.urlopen(invalid_mode_request, timeout=5) - self.assertEqual(ctx.exception.code, 400) + with urllib.request.urlopen(invalid_mode_request, timeout=5) as response: + self.assertEqual(response.status, 200) with urllib.request.urlopen(f"{base_url}/v1/ai/automation/runs?include_events=true", timeout=5) as response: ledger = json.loads(response.read().decode("utf-8"))["ledger"] - self.assertEqual(ledger["summary"]["total_runs"], 2) + self.assertEqual(ledger["summary"]["total_runs"], 3) self.assertEqual(ledger["runs"][0]["task_name"], "platform-health") self.assertIn("events", ledger["runs"][0]) @@ -686,15 +685,6 @@ def test_automation_triage_reports_retryable_incident(self) -> None: "run_id": "incident-123", "mode": "manual", } - blank_mode_request = urllib.request.Request( - f"{base_url}/v1/ai/automation/triage", - data=json.dumps({**payload, "mode": ""}).encode("utf-8"), - method="POST", - headers={"Content-Type": "application/json"}, - ) - with self.assertRaises(urllib.error.HTTPError) as ctx: - urllib.request.urlopen(blank_mode_request, timeout=5) - self.assertEqual(ctx.exception.code, 400) request = urllib.request.Request( f"{base_url}/v1/ai/automation/triage", data=json.dumps(payload).encode("utf-8"), From 531390fc3a35789e234c3cbeb22e9ce3f179fef2 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:49:52 +0800 Subject: [PATCH 72/72] fix: recover unknown automation history after boundary Co-Authored-By: Codex --- service/ai_gateway_service.py | 2 +- tests/test_ai_gateway_automation_control.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/service/ai_gateway_service.py b/service/ai_gateway_service.py index e1bba1db..6365162e 100644 --- a/service/ai_gateway_service.py +++ b/service/ai_gateway_service.py @@ -559,7 +559,7 @@ def _automation_control_snapshot( failure_history_complete = ( not ledger_unavailable and not storage_unavailable - and (not bool(retention.get("history_completeness_unknown")) or (repo_evictions > 0 and repo_history_has_terminal_boundary)) + and (not bool(retention.get("history_completeness_unknown")) or repo_history_has_terminal_boundary) and (repo_evictions <= 0 or repo_history_has_terminal_boundary) ) execution = decide_automation_execution( diff --git a/tests/test_ai_gateway_automation_control.py b/tests/test_ai_gateway_automation_control.py index da77889e..74a9a5b0 100644 --- a/tests/test_ai_gateway_automation_control.py +++ b/tests/test_ai_gateway_automation_control.py @@ -225,7 +225,7 @@ def test_control_snapshot_allows_known_repo_boundary_when_history_unknown(self) "history_completeness_unknown": True, "may_be_truncated": True, "evicted_runs": 1, - "evicted_runs_by_repo": {"quantstrategylab/targetrepo": 1}, + "evicted_runs_by_repo": {"quantstrategylab/targetrepo": 0}, } }, }