From 02b5be8e514325b55389c3be3b2112db00fec217 Mon Sep 17 00:00:00 2001 From: JingWen Fan <106414602+study8677@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:51:51 +0800 Subject: [PATCH 1/2] fix(refresh): stop refresh hanging on slow / reasoning models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes so `ag-refresh` works out-of-the-box with slow reasoning models (e.g. MiniMax-M3) instead of appearing to hang for many minutes: 1. A bare asyncio.wait_for timeout is no longer treated as retryable (_is_retryable_error). Our own per-attempt deadline elapsing means the model is slow/stalling, not transiently failing — retrying the same full-length attempt just multiplied wall-clock by (AG_REFRESH_RETRY_COUNT + 1) (default 4x) for the same outcome. It now falls back immediately. Messaged provider timeouts ("504" / "gateway time-out") stay retryable. 2. Raise the refresh per-attempt timeout defaults, which were tuned for fast models and too low for reasoning models (one call may emit a long reasoning block; conventions is a 3-hop handoff swarm): - AG_REFRESH_AGENT_TIMEOUT_SECONDS 90 -> 300 (conventions swarm) - AG_MODULE_AGENT_TIMEOUT_SECONDS 45 -> 240 (per-module doc) - AG_MAP_AGENT_TIMEOUT_SECONDS 90 -> 240 (map.md) - AG_REGISTRY_TIMEOUT_SECONDS 60 -> 120 A bare timeout no longer retries, so a higher ceiling only costs time if a call genuinely runs that long; fast models finish well under these. Verified end-to-end: a full multi-module refresh completes with real (non-fallback) docs on MiniMax-M3. Adds test_refresh_retry.py and documents the timeouts in engine/.env.example. Co-Authored-By: Claude Opus 4.8 (1M context) --- engine/.env.example | 11 ++++++ .../hub/refresh_pipeline.py | 24 ++++++++---- engine/tests/test_refresh_retry.py | 38 +++++++++++++++++++ 3 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 engine/tests/test_refresh_retry.py diff --git a/engine/.env.example b/engine/.env.example index b3f4fdc1c..0b409c128 100644 --- a/engine/.env.example +++ b/engine/.env.example @@ -36,6 +36,17 @@ # Initial backoff delay in seconds (doubles each attempt). # AG_REFRESH_RETRY_DELAY=1.0 +# ------------------------------------------------------------------- +# Refresh agent timeouts (optional) — per-attempt wall-clock seconds. +# Defaults are sized for slow *reasoning* models (a single call may emit a long +# reasoning/ block); fast models finish well under them. A bare timeout +# is NOT retried (it falls back immediately), so raising these only costs time +# if a call genuinely runs that long. +# AG_REFRESH_AGENT_TIMEOUT_SECONDS=300 # conventions: 3-hop handoff swarm +# AG_MODULE_AGENT_TIMEOUT_SECONDS=240 # per-module knowledge doc +# AG_MAP_AGENT_TIMEOUT_SECONDS=240 # map.md generation over all docs +# AG_REGISTRY_TIMEOUT_SECONDS=120 # module registry + # ------------------------------------------------------------------- # Ask retry policy (optional) # Same-provider retry for transient ask-time failures (timeouts, 5xx, diff --git a/engine/antigravity_engine/hub/refresh_pipeline.py b/engine/antigravity_engine/hub/refresh_pipeline.py index e922c95f2..7a5959a3e 100644 --- a/engine/antigravity_engine/hub/refresh_pipeline.py +++ b/engine/antigravity_engine/hub/refresh_pipeline.py @@ -143,9 +143,16 @@ def _is_retryable_error(exc: Exception) -> bool: Returns: True if the error is retryable. """ - # asyncio.TimeoutError has an empty str() — check type first. - if isinstance(exc, (TimeoutError, asyncio.TimeoutError)): - return True + # A bare asyncio wait_for timeout has an empty str() and means OUR own + # per-attempt deadline elapsed — the model is slow/stalling, not + # transiently failing. Retrying the same full-length attempt just + # multiplies wall-clock by (retries + 1) for the same outcome, which is + # what made refresh appear to hang. Treat a *bare* timeout as NON-retryable + # so the step falls back immediately. A provider-side timeout that carries + # a message (e.g. "gateway time-out"/"504") falls through to the keyword + # check below and stays retryable. + if isinstance(exc, (TimeoutError, asyncio.TimeoutError)) and not str(exc).strip(): + return False msg = str(exc).lower() retryable_keywords = ( "timeout", @@ -332,7 +339,10 @@ async def refresh_pipeline(workspace: Path, quick: bool = False, failed_only: bo print("[2/3] Analyzing with multi-agent swarm...", file=sys.stderr) - refresh_timeout = float(os.environ.get("AG_REFRESH_AGENT_TIMEOUT_SECONDS", "90")) + # Conventions is a 3-hop handoff swarm (ScanAnalyst → ArchitectureReviewer + # → ConventionWriter), so it needs ~3x a single call. Default is generous + # enough for slow reasoning models; fast models finish well under it. + refresh_timeout = float(os.environ.get("AG_REFRESH_AGENT_TIMEOUT_SECONDS", "300")) try: result = await _run_with_retry( Runner.run, agent, prompt, @@ -449,7 +459,7 @@ async def refresh_pipeline(workspace: Path, quick: bool = False, failed_only: bo "OpenAI Agent SDK not found. Install: pip install antigravity-engine" ) from None - module_timeout = float(os.environ.get("AG_MODULE_AGENT_TIMEOUT_SECONDS", "45")) + module_timeout = float(os.environ.get("AG_MODULE_AGENT_TIMEOUT_SECONDS", "240")) # Skip module agents when failed-only mode has no modules to process if modules_filter is not None and not modules_filter: @@ -1902,7 +1912,7 @@ async def _generate_map_md(workspace: Path, model: str) -> str: batches.append(current_batch) map_agent = build_map_agent(model) - map_timeout = float(os.environ.get("AG_MAP_AGENT_TIMEOUT_SECONDS", "90")) + map_timeout = float(os.environ.get("AG_MAP_AGENT_TIMEOUT_SECONDS", "240")) async def _run_map_batch(batch: list[str], batch_idx: int) -> str: prompt = "Create a map.md from these module knowledge documents:\n" + "\n".join(batch) @@ -2100,7 +2110,7 @@ async def _generate_module_registry(workspace: Path, model: str) -> str: model=model, ) - registry_timeout = float(os.environ.get("AG_REGISTRY_TIMEOUT_SECONDS", "60")) + registry_timeout = float(os.environ.get("AG_REGISTRY_TIMEOUT_SECONDS", "120")) result = await _run_with_retry( Runner.run, registry_agent, prompt, timeout=registry_timeout, diff --git a/engine/tests/test_refresh_retry.py b/engine/tests/test_refresh_retry.py new file mode 100644 index 000000000..b159e9e5d --- /dev/null +++ b/engine/tests/test_refresh_retry.py @@ -0,0 +1,38 @@ +"""Regression tests for refresh retry classification. + +A bare ``asyncio.wait_for`` timeout means our own per-attempt deadline elapsed +(the model is slow/stalling). Retrying it just multiplies wall-clock by +(retries + 1) for the same outcome — the behaviour that made refresh appear to +hang. It must be treated as NON-retryable so the step falls back immediately. +Genuine transient provider failures (rate limits, 5xx, network, and +*messaged* gateway timeouts) must still be retried. +""" + +from __future__ import annotations + +import asyncio + +from antigravity_engine.hub.refresh_pipeline import _is_retryable_error + + +def test_bare_wait_for_timeout_is_not_retryable() -> None: + # asyncio.TimeoutError() and TimeoutError() carry an empty message. + assert _is_retryable_error(asyncio.TimeoutError()) is False + assert _is_retryable_error(TimeoutError()) is False + + +def test_messaged_gateway_timeout_is_retryable() -> None: + # A provider-side timeout carries a message and stays retryable. + assert _is_retryable_error(TimeoutError("504 Gateway Time-out")) is True + + +def test_transient_provider_errors_are_retryable() -> None: + assert _is_retryable_error(RuntimeError("connection reset by peer")) is True + assert _is_retryable_error(RuntimeError("rate limit exceeded")) is True + assert _is_retryable_error(RuntimeError("503 Service Unavailable")) is True + assert _is_retryable_error(RuntimeError("network is unreachable")) is True + + +def test_non_transient_errors_are_not_retryable() -> None: + assert _is_retryable_error(ValueError("invalid api key")) is False + assert _is_retryable_error(RuntimeError("bad request: malformed prompt")) is False From bb1153ad92ea201b948948f8aad08662053f45b2 Mon Sep 17 00:00:00 2001 From: JingWen Fan <106414602+study8677@users.noreply.github.com> Date: Thu, 4 Jun 2026 20:05:20 +0800 Subject: [PATCH 2/2] fix(refresh): bump module/map agent timeout defaults to 300s OOTB verification showed per-module knowledge-doc calls still timing out at 240s with MiniMax-M3 (3 modules concurrent) and falling back; the proven value is 300s (same as the conventions swarm). Raise AG_MODULE_AGENT_TIMEOUT_SECONDS and AG_MAP_AGENT_TIMEOUT_SECONDS defaults 240 -> 300 so refresh produces real (non-fallback) docs out of the box. Co-Authored-By: Claude Opus 4.8 (1M context) --- engine/.env.example | 4 ++-- engine/antigravity_engine/hub/refresh_pipeline.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/.env.example b/engine/.env.example index 0b409c128..da35c1656 100644 --- a/engine/.env.example +++ b/engine/.env.example @@ -43,8 +43,8 @@ # is NOT retried (it falls back immediately), so raising these only costs time # if a call genuinely runs that long. # AG_REFRESH_AGENT_TIMEOUT_SECONDS=300 # conventions: 3-hop handoff swarm -# AG_MODULE_AGENT_TIMEOUT_SECONDS=240 # per-module knowledge doc -# AG_MAP_AGENT_TIMEOUT_SECONDS=240 # map.md generation over all docs +# AG_MODULE_AGENT_TIMEOUT_SECONDS=300 # per-module knowledge doc +# AG_MAP_AGENT_TIMEOUT_SECONDS=300 # map.md generation over all docs # AG_REGISTRY_TIMEOUT_SECONDS=120 # module registry # ------------------------------------------------------------------- diff --git a/engine/antigravity_engine/hub/refresh_pipeline.py b/engine/antigravity_engine/hub/refresh_pipeline.py index 7a5959a3e..c99c6e991 100644 --- a/engine/antigravity_engine/hub/refresh_pipeline.py +++ b/engine/antigravity_engine/hub/refresh_pipeline.py @@ -459,7 +459,7 @@ async def refresh_pipeline(workspace: Path, quick: bool = False, failed_only: bo "OpenAI Agent SDK not found. Install: pip install antigravity-engine" ) from None - module_timeout = float(os.environ.get("AG_MODULE_AGENT_TIMEOUT_SECONDS", "240")) + module_timeout = float(os.environ.get("AG_MODULE_AGENT_TIMEOUT_SECONDS", "300")) # Skip module agents when failed-only mode has no modules to process if modules_filter is not None and not modules_filter: @@ -1912,7 +1912,7 @@ async def _generate_map_md(workspace: Path, model: str) -> str: batches.append(current_batch) map_agent = build_map_agent(model) - map_timeout = float(os.environ.get("AG_MAP_AGENT_TIMEOUT_SECONDS", "240")) + map_timeout = float(os.environ.get("AG_MAP_AGENT_TIMEOUT_SECONDS", "300")) async def _run_map_batch(batch: list[str], batch_idx: int) -> str: prompt = "Create a map.md from these module knowledge documents:\n" + "\n".join(batch)