feat: recovery_candidates wired into process_plan_tick + Pipeline 3 telemetry + experiments 008/009

agents/agent_loop.py

@@ -46,6 +46,7 @@
 from plan_schema import (
     Plan, PlanState, DangerLevel, VerifyType, load_plan, save_plan,
 )
+from recovery_candidates import maybe_synthesize_substitute
 
 MC_API_URL = os.getenv("MC_API_URL", "http://localhost:3001")
 EMBODIED_SERVICE_URL = os.getenv("EMBODIED_SERVICE_URL", "http://localhost:7790")
@@ -438,6 +439,39 @@ def process_plan_tick(plan: Plan, now: float) -> tuple[Plan, dict]:
     passed, reason = verify_step(step, embodied_result)
     _log_event("step_verify", step_id=step.id, passed=passed, reason=reason)
 
+    # Path D candidate-generator: when verify fails with bot-side details,
+    # synthesise a deterministic substitute from world state. The body
+    # model emits the substitute via selection (lesson 7 + experiment 009:
+    # 100% with explicit alternatives). If synthesis returns None, fall
+    # through to existing retry-with-backoff. Only fires when retries
+    # remaining ≥ 2 to preserve the last retry budget for the
+    # un-substituted fallback case.
+    if not passed and step.retries < step.max_retries - 1:
+        try:
+            status = fetch_bot_status()
+            nearby = fetch_bot_nearby()
+            inv = fetch_bot_inventory()
+            pos = status.get("position", {})
+            bot_xyz = (int(pos.get("x", 0)), int(pos.get("y", 0)), int(pos.get("z", 0)))
+            substitute_intent = maybe_synthesize_substitute(
+                step, embodied_result,
+                bot_position=bot_xyz,
+                nearby_blocks=nearby.get("blocks", nearby.get("nearby_blocks", [])),
+                inventory=inv,
+            )
+            if substitute_intent:
+                _log_event("substitute_synthesised", step_id=step.id,
+                           original_intent=step.intent[:120],
+                           new_intent=substitute_intent[:120])
+                # Re-run with the substitute intent immediately (this tick),
+                # without persisting the rewrite — Plan stays under Steve's
+                # authority.
+                embodied_result = call_embodied(substitute_intent)
+                passed, reason = verify_step(step, embodied_result)
+                _log_event("substitute_verify", step_id=step.id, passed=passed, reason=reason)
+        except Exception as e:
+            _log_event("substitute_synthesis_failed", step_id=step.id, error=str(e))
+
     if passed:
         plan.current_step += 1
         plan.last_advance_ts = now

agents/embodied-service/index.js

@@ -172,11 +172,23 @@ async function handleIntent(req, res) {
   try {
     parsed = parseGemmaAndyResponse(ollama_result.raw);
   } catch (err) {
+    const raw_text = ollama_result.raw ?? "";
+    // Truncation heuristic: Gemma-Andy's response shape ends with `}`
+    // (outer object close). If the last `}` is missing entirely, or
+    // appears more than 50 chars before the end, treat as truncated.
+    // Using only `}` (not `]`) avoids false negatives when the inner
+    // tool_calls array closes but the outer object never does — an
+    // observed failure mode when num_predict cuts mid-emit.
+    const last_brace = raw_text.lastIndexOf("}");
+    const truncated_heuristic = raw_text.length > 0
+      && (last_brace === -1 || last_brace < raw_text.length - 50);
     logEvent({
       event: "parse_failed",
       context_id,
       error: err.message,
-      raw_excerpt: ollama_result.raw.slice(0, 400),
+      raw_excerpt: raw_text.slice(0, 400),
+      raw_length_chars: raw_text.length,
+      truncated: truncated_heuristic,
     });
     // Mitigation: if the model returned an empty string, synthesize a
     // signal so upstream can act. This is a regression observed in
@@ -250,6 +262,7 @@ async function handleIntent(req, res) {
     operational_risk: parsed.plan.operational_risk,
     tool_call_count: parsed.plan.tool_calls.length,
     had_think: parsed.think != null,
+    think_excerpt: typeof parsed.think === "string" ? parsed.think.slice(0, 500) : null,
     plan: parsed.plan,
     think: parsed.think,
   });
@@ -275,6 +288,7 @@ async function handleIntent(req, res) {
       tool: r.tool,
       ok: r.ok,
       error_type: r.error_type,
+      details: r.details ?? null,
     });
     if (!r.ok) break;
   }

agents/embodied-service/primitives_lab/experiments/008_recovery_generalization.yaml

@@ -0,0 +1,83 @@
+# 008 — Recovery generalization (Pipeline 2 validation)
+#
+# Lesson 7 (experiments/007_in_intent_replan.yaml) validated that an
+# in-intent narrative recovery shifts the model's plan 90-100% for the
+# block-substitution failure mode (oak_log → oak_planks). Pipeline 2 of
+# the 2026-05-10 plan generalises that recovery into the embodied_plan
+# Python handler, which now triggers automatically on ANY bot_action_failed
+# that carries a `details` string.
+#
+# This experiment validates that the in-intent narrative shape works
+# across the four distinct failure modes the place_block dispatcher emits:
+#   1. target space occupied (e.g., leaf_litter at the target coord)
+#   2. no solid adjacent block to place against (target is open air)
+#   3. target equals bot's own position (bot occupies the space)
+#   4. inventory missing the requested block name
+#
+# Note: this experiment does NOT exercise the Python handler's auto-retry
+# logic. It tests whether the BODY MODEL responds correctly to an intent
+# that EMBEDS the failure detail. If this generalises, the handler's
+# auto-retry (which composes such intents synchronously) is sound.
+
+id: 008-recovery-generalization
+hypothesis: "In-intent narrative recovery generalises beyond block-substitution to all bot_action_failed modes that carry actionable info"
+fixture: forest_with_player
+samples_per_variant: 5
+
+variants:
+  - id: occupied_recovery
+    primitives:
+      intent: |
+        We tried to Place 1 oak_planks at coordinates (7, 65, 35), but
+        place_block failed with error_type=bot_action_failed. Can't place
+        oak_planks at 7, 65, 35: target space is occupied by leaf_litter.
+        Compose a new plan that achieves the same outcome using the
+        actually-available state. Do not re-emit the failing action.
+
+  - id: no_neighbor_recovery
+    primitives:
+      intent: |
+        We tried to Place 1 oak_planks at coordinates (7, 67, 37), but
+        place_block failed with error_type=bot_action_failed. Can't place
+        oak_planks at 7, 67, 37: no solid adjacent block to place against.
+        Choose an empty space next to/above an existing block.
+        Compose a new plan that achieves the same outcome using the
+        actually-available state. Do not re-emit the failing action.
+
+  - id: bot_self_recovery
+    primitives:
+      intent: |
+        We tried to Place 1 oak_planks at coordinates (4, 65, 38), but
+        place_block failed with error_type=bot_action_failed. The bot is
+        currently standing at (4, 65, 38) and cannot place a block in the
+        space it occupies. Choose adjacent coordinates instead.
+        Compose a new plan that achieves the same outcome using the
+        actually-available state. Do not re-emit the failing action.
+
+  - id: missing_inventory_recovery
+    primitives:
+      intent: |
+        We tried to Place 1 cobblestone at coordinates (5, 65, 38), but
+        place_block failed with error_type=bot_action_failed. No
+        cobblestone in inventory. Inventory has: oak_planks(40),
+        leaf_litter(5), stick(2). Use an available block instead.
+        Compose a new plan that achieves the same outcome using the
+        actually-available state. Do not re-emit the failing action.
+
+expectations:
+  must_invoke_embodied_plan: true
+  tool_calls_must_include_any_of: [place_block, scan_nearby, get_inventory, ask_clarification]
+  max_elapsed_seconds: 30
+  operational_risk_in: [none, low, medium]
+
+metrics_to_capture:
+  - success_rate
+  - latency_p50
+  - tool_call_count
+  - mitigation_rate
+  - tool_calls_emitted
+
+# Reading: passes = model emitted SOMETHING in the allowed-tools set.
+# But the load-bearing analytical question is: did the model RE-EMIT
+# the same failing position? Tooled by manual analysis after run
+# (runner.py captures tool_calls.arguments per-sample post round-2 fix).

agents/embodied-service/primitives_lab/experiments/009_explicit_substitute_recovery.yaml

@@ -0,0 +1,94 @@
+# 009 — Explicit substitute recovery (Codex audit experiment 1)
+#
+# Codex audit of the lessons-007/008 contrast found that the body model
+# is good at SELECTION (when given a substitute, lesson 7 → 9-10/10) but
+# bad at SEARCH (when asked to invent a substitute from a negation,
+# lesson 8 → 0-2/5). The proposed dispatcher-classification plan was
+# challenged: it's still asking the model to do search-like reasoning
+# over a directive template.
+#
+# This experiment tests Codex's pick (D): if the recovery intent NAMES
+# a specific alternative coord/block (i.e., the dispatcher synthesised
+# the candidate), does the body model recover at lesson-7 levels across
+# all four 008 failure modes — or does the failure-mode shape itself
+# matter beyond candidate availability?
+#
+# Hypothesis (D): explicit substitutes recover at 9-10/10 across all
+# four modes. Confirms D, weakens C.
+# Anti-hypothesis: even with explicit substitutes, some modes (e.g.,
+# bot_self) still fail because the prose still mentions the failing
+# coord. Would weaken D.
+#
+# E2E validation: each variant's substitute coord is east of the
+# existing wall at x=5 (built earlier in session). The first sample
+# of each variant can successfully place; samples 2-5 will hit
+# "target space is occupied" because the substitute coord is now
+# filled. Plan match (Andy emitted the substitute coord, not the
+# failing target) is the primary metric; e2e success of sample 1
+# confirms the full Hermes→Andy→bot chain works given good intents.
+
+id: 009-explicit-substitute-recovery
+hypothesis: "Explicit substitute coords/blocks in the recovery intent achieve lesson-7-level recovery (90-100%) across all four 008 failure modes"
+fixture: forest_with_player
+samples_per_variant: 5
+
+variants:
+  - id: occupied_with_substitute
+    primitives:
+      intent: |
+        We tried to place 1 oak_planks at coordinates (7, 65, 35), but
+        place_block failed with error_type=bot_action_failed. The target
+        space was occupied by leaf_litter. Place 1 oak_planks at
+        coordinates (6, 65, 38) instead — that coordinate is air and
+        has the existing wall block at (5, 65, 38) as a placement
+        neighbor.
+
+  - id: no_neighbor_with_substitute
+    primitives:
+      intent: |
+        We tried to place 1 oak_planks at coordinates (7, 67, 37), but
+        place_block failed with error_type=bot_action_failed. There was
+        no solid block adjacent to that coordinate to anchor against.
+        Place 1 oak_planks at coordinates (6, 66, 38) instead — that
+        coordinate is air and has the existing wall block at (5, 66, 38)
+        as a placement neighbor.
+
+  - id: bot_self_with_substitute
+    primitives:
+      intent: |
+        We tried to place 1 oak_planks at coordinates (4, 65, 38), but
+        place_block failed with error_type=bot_action_failed. The bot
+        was standing at that coordinate and cannot place a block in the
+        space it occupies. Place 1 oak_planks at coordinates
+        (6, 67, 38) instead — that coordinate is air, far from the bot,
+        and has the existing wall block at (5, 67, 38) as a placement
+        neighbor.
+
+  - id: missing_inventory_with_substitute
+    primitives:
+      intent: |
+        We tried to place 1 cobblestone at coordinates (5, 65, 38), but
+        place_block failed with error_type=bot_action_failed. There is
+        no cobblestone in inventory. Inventory has oak_planks(many).
+        Place 1 oak_planks at coordinates (6, 68, 38) instead — that
+        coordinate is air and has the existing wall block at
+        (5, 68, 38) as a placement neighbor.
+
+expectations:
+  must_invoke_embodied_plan: true
+  tool_calls_must_include_any_of: [place_block]
+  max_elapsed_seconds: 30
+  operational_risk_in: [none, low, medium]
+
+metrics_to_capture:
+  - success_rate
+  - latency_p50
+  - tool_call_count
+  - mitigation_rate
+  - tool_calls_emitted
+
+# Reading: the load-bearing analytical question is whether place_block
+# in tool_calls.arguments.position matches the SUBSTITUTE coord
+# (variant-specific) and not the FAILING-TARGET coord. Manual analysis
+# post-run via the per-sample tool_calls capture (post-round-2 runner
+# fix). E2E validation: was sample 1 of each variant ok=true?