diff --git a/engine/.env.example b/engine/.env.example index b3f4fdc1c..da35c1656 100644 --- a/engine/.env.example +++ b/engine/.env.example @@ -36,6 +36,17 @@ # Initial backoff delay in seconds (doubles each attempt). # AG_REFRESH_RETRY_DELAY=1.0 +# ------------------------------------------------------------------- +# Refresh agent timeouts (optional) — per-attempt wall-clock seconds. +# Defaults are sized for slow *reasoning* models (a single call may emit a long +# reasoning/ block); fast models finish well under them. A bare timeout +# is NOT retried (it falls back immediately), so raising these only costs time +# if a call genuinely runs that long. +# AG_REFRESH_AGENT_TIMEOUT_SECONDS=300 # conventions: 3-hop handoff swarm +# AG_MODULE_AGENT_TIMEOUT_SECONDS=300 # per-module knowledge doc +# AG_MAP_AGENT_TIMEOUT_SECONDS=300 # map.md generation over all docs +# AG_REGISTRY_TIMEOUT_SECONDS=120 # module registry + # ------------------------------------------------------------------- # Ask retry policy (optional) # Same-provider retry for transient ask-time failures (timeouts, 5xx, diff --git a/engine/antigravity_engine/hub/refresh_pipeline.py b/engine/antigravity_engine/hub/refresh_pipeline.py index e922c95f2..c99c6e991 100644 --- a/engine/antigravity_engine/hub/refresh_pipeline.py +++ b/engine/antigravity_engine/hub/refresh_pipeline.py @@ -143,9 +143,16 @@ def _is_retryable_error(exc: Exception) -> bool: Returns: True if the error is retryable. """ - # asyncio.TimeoutError has an empty str() — check type first. - if isinstance(exc, (TimeoutError, asyncio.TimeoutError)): - return True + # A bare asyncio wait_for timeout has an empty str() and means OUR own + # per-attempt deadline elapsed — the model is slow/stalling, not + # transiently failing. Retrying the same full-length attempt just + # multiplies wall-clock by (retries + 1) for the same outcome, which is + # what made refresh appear to hang. Treat a *bare* timeout as NON-retryable + # so the step falls back immediately. A provider-side timeout that carries + # a message (e.g. "gateway time-out"/"504") falls through to the keyword + # check below and stays retryable. + if isinstance(exc, (TimeoutError, asyncio.TimeoutError)) and not str(exc).strip(): + return False msg = str(exc).lower() retryable_keywords = ( "timeout", @@ -332,7 +339,10 @@ async def refresh_pipeline(workspace: Path, quick: bool = False, failed_only: bo print("[2/3] Analyzing with multi-agent swarm...", file=sys.stderr) - refresh_timeout = float(os.environ.get("AG_REFRESH_AGENT_TIMEOUT_SECONDS", "90")) + # Conventions is a 3-hop handoff swarm (ScanAnalyst → ArchitectureReviewer + # → ConventionWriter), so it needs ~3x a single call. Default is generous + # enough for slow reasoning models; fast models finish well under it. + refresh_timeout = float(os.environ.get("AG_REFRESH_AGENT_TIMEOUT_SECONDS", "300")) try: result = await _run_with_retry( Runner.run, agent, prompt, @@ -449,7 +459,7 @@ async def refresh_pipeline(workspace: Path, quick: bool = False, failed_only: bo "OpenAI Agent SDK not found. Install: pip install antigravity-engine" ) from None - module_timeout = float(os.environ.get("AG_MODULE_AGENT_TIMEOUT_SECONDS", "45")) + module_timeout = float(os.environ.get("AG_MODULE_AGENT_TIMEOUT_SECONDS", "300")) # Skip module agents when failed-only mode has no modules to process if modules_filter is not None and not modules_filter: @@ -1902,7 +1912,7 @@ async def _generate_map_md(workspace: Path, model: str) -> str: batches.append(current_batch) map_agent = build_map_agent(model) - map_timeout = float(os.environ.get("AG_MAP_AGENT_TIMEOUT_SECONDS", "90")) + map_timeout = float(os.environ.get("AG_MAP_AGENT_TIMEOUT_SECONDS", "300")) async def _run_map_batch(batch: list[str], batch_idx: int) -> str: prompt = "Create a map.md from these module knowledge documents:\n" + "\n".join(batch) @@ -2100,7 +2110,7 @@ async def _generate_module_registry(workspace: Path, model: str) -> str: model=model, ) - registry_timeout = float(os.environ.get("AG_REGISTRY_TIMEOUT_SECONDS", "60")) + registry_timeout = float(os.environ.get("AG_REGISTRY_TIMEOUT_SECONDS", "120")) result = await _run_with_retry( Runner.run, registry_agent, prompt, timeout=registry_timeout, diff --git a/engine/tests/test_refresh_retry.py b/engine/tests/test_refresh_retry.py new file mode 100644 index 000000000..b159e9e5d --- /dev/null +++ b/engine/tests/test_refresh_retry.py @@ -0,0 +1,38 @@ +"""Regression tests for refresh retry classification. + +A bare ``asyncio.wait_for`` timeout means our own per-attempt deadline elapsed +(the model is slow/stalling). Retrying it just multiplies wall-clock by +(retries + 1) for the same outcome — the behaviour that made refresh appear to +hang. It must be treated as NON-retryable so the step falls back immediately. +Genuine transient provider failures (rate limits, 5xx, network, and +*messaged* gateway timeouts) must still be retried. +""" + +from __future__ import annotations + +import asyncio + +from antigravity_engine.hub.refresh_pipeline import _is_retryable_error + + +def test_bare_wait_for_timeout_is_not_retryable() -> None: + # asyncio.TimeoutError() and TimeoutError() carry an empty message. + assert _is_retryable_error(asyncio.TimeoutError()) is False + assert _is_retryable_error(TimeoutError()) is False + + +def test_messaged_gateway_timeout_is_retryable() -> None: + # A provider-side timeout carries a message and stays retryable. + assert _is_retryable_error(TimeoutError("504 Gateway Time-out")) is True + + +def test_transient_provider_errors_are_retryable() -> None: + assert _is_retryable_error(RuntimeError("connection reset by peer")) is True + assert _is_retryable_error(RuntimeError("rate limit exceeded")) is True + assert _is_retryable_error(RuntimeError("503 Service Unavailable")) is True + assert _is_retryable_error(RuntimeError("network is unreachable")) is True + + +def test_non_transient_errors_are_not_retryable() -> None: + assert _is_retryable_error(ValueError("invalid api key")) is False + assert _is_retryable_error(RuntimeError("bad request: malformed prompt")) is False