|
| 1 | +"""Health-driven execution decisions for automation scheduling.""" |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +import json |
| 6 | +import os |
| 7 | +from pathlib import Path |
| 8 | +from typing import Any |
| 9 | + |
| 10 | +from service.automation_run_ledger import CONTROL_ESCALATE, CONTROL_PAUSE_AUTO_FIX, CONTROL_REVIEW_ONLY |
| 11 | +from service.quota import recommend_model |
| 12 | + |
| 13 | +EXECUTION_RUN = "run" |
| 14 | +EXECUTION_REVIEW_ONLY = "review_only" |
| 15 | +EXECUTION_DEFER = "defer" |
| 16 | +EXECUTION_HUMAN_REVIEW = "human_review" |
| 17 | + |
| 18 | +MODE_REVIEW_AND_FIX = "review_and_fix" |
| 19 | +MODE_REVIEW_ONLY = "review_only" |
| 20 | + |
| 21 | +AUTONOMY_MANUAL = "manual" |
| 22 | +AUTONOMY_REVIEW_ONLY = "review_only" |
| 23 | +AUTONOMY_AUTO_PR = "auto_pr" |
| 24 | +AUTONOMY_AUTO_MERGE = "auto_merge" |
| 25 | +AUTONOMY_ORDER = (AUTONOMY_MANUAL, AUTONOMY_REVIEW_ONLY, AUTONOMY_AUTO_PR, AUTONOMY_AUTO_MERGE) |
| 26 | +AUTONOMY_RANK = {level: index for index, level in enumerate(AUTONOMY_ORDER)} |
| 27 | + |
| 28 | +DEFAULT_MAX_CONSECUTIVE_FAILURES = 3 |
| 29 | +DEFAULT_LOW_COST_MODEL = "gpt-5.4-mini" |
| 30 | +EXECUTION_POLICY_PATH_ENV = "CODEX_AUDIT_SERVICE_EXECUTION_POLICY_PATH" |
| 31 | +QUOTA_STATUS_SEVERITY = { |
| 32 | + "ok": 0, |
| 33 | + "healthy": 0, |
| 34 | + "unknown": 1, |
| 35 | + "unavailable": 1, |
| 36 | + "low": 2, |
| 37 | + "constrained": 2, |
| 38 | + "exhausted": 3, |
| 39 | + "blocked": 3, |
| 40 | +} |
| 41 | + |
| 42 | + |
| 43 | +def _normalize_status(value: Any, default: str = "unknown") -> str: |
| 44 | + if isinstance(value, dict): |
| 45 | + value = value.get("status", default) |
| 46 | + return str(value or default).strip().lower() |
| 47 | + |
| 48 | + |
| 49 | +def _normalize_quota_status(value: Any, default: str = "unknown") -> str: |
| 50 | + statuses = [_normalize_status(value, "")] |
| 51 | + if isinstance(value, dict) and isinstance(value.get("quota"), dict): |
| 52 | + statuses.append(_normalize_status(value["quota"], "")) |
| 53 | + normalized = [status for status in statuses if status] |
| 54 | + if not normalized: |
| 55 | + return default |
| 56 | + return max(normalized, key=lambda status: QUOTA_STATUS_SEVERITY.get(status, 1)) |
| 57 | + |
| 58 | + |
| 59 | +def _normalize_mode(value: str) -> str: |
| 60 | + return MODE_REVIEW_AND_FIX if str(value or "").strip() == MODE_REVIEW_AND_FIX else MODE_REVIEW_ONLY |
| 61 | + |
| 62 | + |
| 63 | +def _normalize_autonomy(value: Any, default: str = AUTONOMY_AUTO_PR) -> str: |
| 64 | + level = str(value or default).strip().lower() |
| 65 | + return level if level in AUTONOMY_RANK else default |
| 66 | + |
| 67 | + |
| 68 | +def _repo_from_run(run: dict[str, Any]) -> str: |
| 69 | + metadata = run.get("metadata") if isinstance(run.get("metadata"), dict) else {} |
| 70 | + return str(metadata.get("source_repository") or metadata.get("repository") or "") |
| 71 | + |
| 72 | + |
| 73 | +def load_execution_policy(path: Path | None = None) -> dict[str, Any]: |
| 74 | + """Load service-owned execution policy for repo autonomy thresholds.""" |
| 75 | + if path is None: |
| 76 | + configured = os.environ.get(EXECUTION_POLICY_PATH_ENV, "").strip() |
| 77 | + if not configured: |
| 78 | + return {} |
| 79 | + path = Path(configured).expanduser() |
| 80 | + try: |
| 81 | + payload = json.loads(path.read_text(encoding="utf-8")) |
| 82 | + except (OSError, json.JSONDecodeError): |
| 83 | + return {} |
| 84 | + return payload if isinstance(payload, dict) else {} |
| 85 | + |
| 86 | + |
| 87 | +def repo_execution_policy(repo: str, policy: dict[str, Any] | None = None) -> dict[str, Any]: |
| 88 | + """Merge default and repo-specific execution policy without trusting repo checkouts.""" |
| 89 | + raw = policy if isinstance(policy, dict) else {} |
| 90 | + defaults = raw.get("default") if isinstance(raw.get("default"), dict) else {} |
| 91 | + repositories = raw.get("repositories") if isinstance(raw.get("repositories"), dict) else {} |
| 92 | + override = repositories.get(repo) if isinstance(repositories.get(repo), dict) else {} |
| 93 | + return {**defaults, **override} |
| 94 | + |
| 95 | + |
| 96 | +def consecutive_failure_count( |
| 97 | + runs: list[dict[str, Any]], |
| 98 | + *, |
| 99 | + repo: str, |
| 100 | + task_name: str = "", |
| 101 | +) -> int: |
| 102 | + """Count latest consecutive failed runs for one repo/task from newest-first runs.""" |
| 103 | + count = 0 |
| 104 | + for run in runs: |
| 105 | + if not isinstance(run, dict): |
| 106 | + continue |
| 107 | + if repo and _repo_from_run(run) != repo: |
| 108 | + continue |
| 109 | + if task_name and str(run.get("task_name") or "") != task_name: |
| 110 | + continue |
| 111 | + state = str(run.get("task_state") or "").strip().lower() |
| 112 | + if state == "failed": |
| 113 | + count += 1 |
| 114 | + continue |
| 115 | + if state: |
| 116 | + break |
| 117 | + return count |
| 118 | + |
| 119 | + |
| 120 | +def decide_automation_execution( |
| 121 | + *, |
| 122 | + repo: str, |
| 123 | + task_name: str = "", |
| 124 | + requested_mode: str = MODE_REVIEW_AND_FIX, |
| 125 | + requested_provider: str = "auto", |
| 126 | + requested_model: str = "", |
| 127 | + control_action: str = CONTROL_REVIEW_ONLY, |
| 128 | + service_health: Any = "", |
| 129 | + quota_status: Any = "", |
| 130 | + org_health_status: Any = "", |
| 131 | + recent_runs: list[dict[str, Any]] | None = None, |
| 132 | + policy: dict[str, Any] | None = None, |
| 133 | +) -> dict[str, Any]: |
| 134 | + """Produce a safe execution decision from health, quota, failures, and repo policy.""" |
| 135 | + repo_policy = repo_execution_policy(repo, policy) |
| 136 | + max_autonomy = _normalize_autonomy(repo_policy.get("max_autonomy"), AUTONOMY_AUTO_PR) |
| 137 | + max_failures = int(repo_policy.get("max_consecutive_failures") or DEFAULT_MAX_CONSECUTIVE_FAILURES) |
| 138 | + low_cost_model = str(repo_policy.get("low_cost_model") or DEFAULT_LOW_COST_MODEL) |
| 139 | + quota_low_behavior = str(repo_policy.get("quota_low_behavior") or "low_cost_model").strip().lower() |
| 140 | + |
| 141 | + effective_mode = _normalize_mode(requested_mode) |
| 142 | + effective_provider = str(requested_provider or "auto").strip().lower() or "auto" |
| 143 | + effective_model = str(requested_model or "").strip() |
| 144 | + action = EXECUTION_RUN |
| 145 | + reasons: list[str] = [] |
| 146 | + human_review_required = False |
| 147 | + defer = False |
| 148 | + |
| 149 | + service = _normalize_status(service_health) |
| 150 | + quota = _normalize_quota_status(quota_status) |
| 151 | + org_health = _normalize_status(org_health_status) |
| 152 | + failures = consecutive_failure_count(recent_runs or [], repo=repo, task_name=task_name) |
| 153 | + |
| 154 | + if AUTONOMY_RANK[max_autonomy] <= AUTONOMY_RANK[AUTONOMY_REVIEW_ONLY]: |
| 155 | + effective_mode = MODE_REVIEW_ONLY |
| 156 | + human_review_required = True |
| 157 | + reasons.append(f"repo max autonomy is {max_autonomy}") |
| 158 | + if max_autonomy == AUTONOMY_MANUAL: |
| 159 | + action = EXECUTION_HUMAN_REVIEW |
| 160 | + |
| 161 | + if failures >= max_failures: |
| 162 | + action = EXECUTION_HUMAN_REVIEW |
| 163 | + effective_mode = MODE_REVIEW_ONLY |
| 164 | + human_review_required = True |
| 165 | + reasons.append(f"consecutive failures reached {failures}/{max_failures}") |
| 166 | + |
| 167 | + if control_action in {CONTROL_REVIEW_ONLY, CONTROL_PAUSE_AUTO_FIX, CONTROL_ESCALATE}: |
| 168 | + effective_mode = MODE_REVIEW_ONLY |
| 169 | + human_review_required = True |
| 170 | + reasons.append(f"runtime control action is {control_action}") |
| 171 | + if control_action == CONTROL_ESCALATE: |
| 172 | + action = EXECUTION_HUMAN_REVIEW |
| 173 | + |
| 174 | + if service == "degraded" or org_health == "degraded": |
| 175 | + effective_mode = MODE_REVIEW_ONLY |
| 176 | + human_review_required = True |
| 177 | + reasons.append("health degraded; forcing review_only") |
| 178 | + if service == "unhealthy" or org_health == "unhealthy": |
| 179 | + action = EXECUTION_HUMAN_REVIEW |
| 180 | + effective_mode = MODE_REVIEW_ONLY |
| 181 | + human_review_required = True |
| 182 | + reasons.append("health unhealthy; forcing human review") |
| 183 | + |
| 184 | + if quota in {"low", "constrained"}: |
| 185 | + effective_model = effective_model or low_cost_model or recommend_model(0.0) |
| 186 | + if quota_low_behavior == "defer": |
| 187 | + action = EXECUTION_DEFER |
| 188 | + defer = True |
| 189 | + effective_mode = MODE_REVIEW_ONLY |
| 190 | + human_review_required = True |
| 191 | + reasons.append(f"quota status is {quota}; deferring automation") |
| 192 | + else: |
| 193 | + reasons.append(f"quota status is {quota}; recommending low-cost model") |
| 194 | + elif quota in {"exhausted", "blocked"}: |
| 195 | + action = EXECUTION_DEFER |
| 196 | + defer = True |
| 197 | + effective_mode = MODE_REVIEW_ONLY |
| 198 | + human_review_required = True |
| 199 | + reasons.append(f"quota status is {quota}; deferring automation") |
| 200 | + |
| 201 | + return { |
| 202 | + "action": action, |
| 203 | + "repo": repo, |
| 204 | + "task_name": task_name, |
| 205 | + "requested_mode": _normalize_mode(requested_mode), |
| 206 | + "effective_mode": effective_mode, |
| 207 | + "requested_provider": requested_provider, |
| 208 | + "effective_provider": effective_provider, |
| 209 | + "requested_model": requested_model, |
| 210 | + "effective_model": effective_model, |
| 211 | + "max_autonomy": max_autonomy, |
| 212 | + "consecutive_failures": failures, |
| 213 | + "max_consecutive_failures": max_failures, |
| 214 | + "human_review_required": human_review_required, |
| 215 | + "auto_fix_allowed": action == EXECUTION_RUN and effective_mode == MODE_REVIEW_AND_FIX and not human_review_required, |
| 216 | + "defer": defer, |
| 217 | + "reasons": reasons or ["execution allowed"], |
| 218 | + } |
0 commit comments