From bcc5280c9b87d349bdb9268e730da626b1fb31fc Mon Sep 17 00:00:00 2001 From: danielgap Date: Tue, 22 Sep 2026 23:58:49 +0200 Subject: [PATCH 1/4] test(orchestrator): reproduce the parent-inline skill contract gap (#348) Deterministic baseline for issue #348 on unmodified main: a project skill with an ## Output Contract is indexed by the skill registry, yet the composed parent prompt binds only subagents to read the exact indexed SKILL.md. No parent-inline read obligation exists and the contract body never reaches parent context, so contract markers cannot reach parent-inline results. Offline runtime harness per tests/asset-installation-runtime.test.ts: scoped-env child boots the real SDK session (no network, in-memory settings), forces /skill-registry:refresh, asserts indexing, then captures the composed parent prompt via __testing.buildGentlePrompt with a fidelity pin proving the render went through the composition pipeline. The two gap assertions are guarded flip helpers for the fix slice. Design note: issues/348#issuecomment-5784664642 --- ...rchestrator-inline-skill-execution.test.ts | 295 ++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 tests/orchestrator-inline-skill-execution.test.ts diff --git a/tests/orchestrator-inline-skill-execution.test.ts b/tests/orchestrator-inline-skill-execution.test.ts new file mode 100644 index 000000000..8cf140a89 --- /dev/null +++ b/tests/orchestrator-inline-skill-execution.test.ts @@ -0,0 +1,295 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +// --------------------------------------------------------------------------- +// gentle-shell#348 — slice 1 (repro): the parent-inline skill contract gap. +// +// Deterministic reproduction, expected to PASS on unmodified main: +// +// 1. A project skill with an `## Output Contract` body is indexed by the +// gentle-ai skill registry (discovery succeeds — `.atl/skill-registry.md` +// lists it with scope `project`). +// 2. The composed parent prompt (the exact `buildGentlePrompt` composition the +// primary session's `before_agent_start` handler injects) only binds +// SUBAGENTS to read the exact indexed `SKILL.md` ("subagents read those +// `SKILL.md` files first"). It imposes no parent-inline read obligation: +// there is no "Parent inline execution" section and no wording binding the +// parent's own inline path to read the indexed `SKILL.md` in-session. +// 3. Consequently the skill's contract body marker can never reach +// parent-inline results through the composed prompt. +// +// Slice 2 must flip exactly the two guarded helpers below: +// `assertParentPromptHasNoInlineSkillReadObligation` (the c-check) and +// `assertParentPromptOmitsSkillContractBody` (the d-check). +// +// Harness: same accepted offline runtime pattern as +// tests/asset-installation-runtime.test.ts — the parent test builds a throwaway +// HOME + agent dir + project (all under one mkdtemp root), spawns this file as +// a child with a fully scoped env (no network, in-memory settings), and the +// child boots the real SDK session with the gentle-ai extension active. +// --------------------------------------------------------------------------- + +const FIXTURE_MARKER = "FIXTURE348-CONTRACT-MARK"; +const FIXTURE_SKILL_NAME = "fixture-contract-probe"; +const CHILD_FLAG = "GENTLE_PI_348_INLINE_PROBE_CHILD"; + +const FIXTURE_SKILL_SOURCE = `--- +name: ${FIXTURE_SKILL_NAME} +description: Probe skill for gentle-shell#348 inline contract reproduction. +--- + +# Fixture Contract Probe + +## Output Contract + +Every result produced under this skill MUST include the literal marker +${FIXTURE_MARKER} on its own line. This contract lives only in the indexed +SKILL.md body; the registry index carries name, description, and path — +never this body. +`; + +function extensionSourceUrl(name: string): string { + return JSON.stringify(new URL(`../extensions/${name}.ts`, import.meta.url).href); +} + +function buildShimSource(): string { + // Loads only what the repro needs from the packaged extensions directory: + // the gentle-ai extension (primary-session orchestrator prompt + session + // lifecycle) and the skill-registry extension (project skill discovery). + return ` +import { createGentleAiExtension } from ${extensionSourceUrl("gentle-ai")}; +import skillRegistry from ${extensionSourceUrl("skill-registry")}; +export default function (pi) { + createGentleAiExtension({ nativeReviewCli: null, candidateViews: null, processEnv: {} })(pi); + skillRegistry(pi); +} +`; +} + +// --- guarded flip targets for slice 2 ------------------------------------- + +// c-check: the composed parent prompt must not bind the parent's OWN inline +// execution path to reading the indexed `SKILL.md` in-session. On main only +// subagents owe that read (the asymmetry under test). Slice 2 inverts this +// helper (or narrows its patterns) once the prompt carries the obligation. +function assertParentPromptHasNoInlineSkillReadObligation(prompt: string): void { + const patterns: readonly RegExp[] = [ + /parent[ -]inline[ -]execution/i, + /inline[^.\n]{0,120}`SKILL\.md`/i, + /`SKILL\.md`[^.\n]{0,120}inline/i, + ]; + for (const pattern of patterns) { + const match = pattern.exec(prompt); + const line = match === null ? "" : prompt.split("\n").find((l) => pattern.test(l)) ?? ""; + assert.ok( + match === null, + `composed parent prompt binds the inline path to SKILL.md reading via /${pattern.source}/: ${JSON.stringify(line.trim())}`, + ); + } +} + +// d-check: the fixture skill's contract body never reaches the composed parent +// prompt (the registry indexes metadata only). Slice 2 flips this helper if +// contract bodies start being delivered inline; otherwise it stays a guard. +function assertParentPromptOmitsSkillContractBody(prompt: string, marker: string): void { + assert.ok( + !prompt.includes(marker), + `composed parent prompt leaked the indexed SKILL.md contract body (marker ${marker})`, + ); +} + +// --------------------------------------------------------------------------- + +async function proveInlineSkillContractGap(): Promise { + const { + createAgentSessionFromServices, + createAgentSessionRuntime, + createAgentSessionServices, + ModelRuntime, + SessionManager, + SettingsManager, + } = await import("@earendil-works/pi-coding-agent"); + + const projectDir = process.env.GENTLE_PI_348_PROJECT!; + const shimPath = process.env.GENTLE_PI_348_SHIM!; + const agentDir = process.env.PI_CODING_AGENT_DIR!; + const cwd = projectDir; + + const modelRuntime = await ModelRuntime.create({ + authPath: join(agentDir, "empty-auth.json"), + modelsPath: null, + modelsStorePath: join(agentDir, "models-store.json"), + allowModelNetwork: false, + }); + + const runtime = await createAgentSessionRuntime( + async ({ cwd, sessionManager, sessionStartEvent }) => { + const services = await createAgentSessionServices({ + cwd, + agentDir, + modelRuntime, + settingsManager: SettingsManager.inMemory({ + retry: { enabled: false }, + compaction: { enabled: false }, + }), + resourceLoaderOptions: { + noExtensions: true, + noSkills: true, + noPromptTemplates: true, + noThemes: true, + noContextFiles: true, + additionalExtensionPaths: [shimPath], + }, + }); + // e-check: the real session boots with zero extension errors and + // zero diagnostics, so any prompt-shape finding below is not an + // artifact of a broken boot. + assert.deepEqual(services.resourceLoader.getExtensions().errors, []); + assert.deepEqual(services.diagnostics, []); + return { + ...(await createAgentSessionFromServices({ + services, + sessionManager, + sessionStartEvent, + tools: ["read", "bash"], + })), + services, + diagnostics: services.diagnostics, + }; + }, + { cwd, agentDir, sessionManager: SessionManager.inMemory(cwd) }, + ); + + try { + await runtime.session.bindExtensions({ mode: "print" }); + const session = runtime.session; + + // a-check (discovery): force the live extension to regenerate the + // registry, then prove the fixture project skill is indexed. The slash + // command runs without a model turn, exactly like the accepted harness + // pattern's /gentle:install-sdd step. + await session.prompt("/skill-registry:refresh"); + assert.equal(session.messages.length, 0, "slash activation must not start a model turn"); + + const registryPath = join(cwd, ".atl", "skill-registry.md"); + assert.ok(existsSync(registryPath), "skill registry must be regenerated in the project"); + const registry = readFileSync(registryPath, "utf8"); + assert.ok( + registry.includes(`\`${FIXTURE_SKILL_NAME}\``), + `registry must index the fixture skill: ${FIXTURE_SKILL_NAME}`, + ); + assert.ok( + registry.includes(`skills/${FIXTURE_SKILL_NAME}/SKILL.md`), + "registry must carry the fixture SKILL.md path", + ); + assert.match(registry, /\| project \|/, "fixture skill must be scoped to the project"); + + // Fixture self-check: the contract marker lives in the indexed body. + const skillBody = readFileSync( + join(cwd, "skills", FIXTURE_SKILL_NAME, "SKILL.md"), + "utf8", + ); + assert.ok(skillBody.includes(FIXTURE_MARKER), "fixture must carry the contract marker"); + assert.ok( + !registry.includes(FIXTURE_MARKER), + "registry index must not inline skill bodies (it carries metadata only)", + ); + + // Composed parent prompt capture: `buildGentlePrompt` is the exact + // composition the primary session's `before_agent_start` handler + // injects (persona, cwd, active tool names, RDD status line). With + // `nativeReviewCli: null` the resolved RDD line equals the default + // worst-case "unknown" line, so this call reproduces the boot + // composition for the primary session byte-for-byte. The module + // instance is shared with the shim (same absolute path), and + // `getOrchestratorPrompt` memoization is per background-policy key — + // irrelevant here because this repro asserts prompt content, not + // per-cwd variance. + const { __testing } = await import("../extensions/gentle-ai.ts"); + const toolNames = session.agent.state.tools.map((tool) => tool.name); + const prompt = __testing.buildGentlePrompt("gentleman", cwd, toolNames); + + // Fidelity pin: the render actually went through the composition + // pipeline (assets root substitution + background/RDD status block). + assert.match( + prompt, + /Receipt-driven development: unknown \(native status unavailable\)/, + "captured prompt must be the composed render, not raw asset bytes", + ); + assert.doesNotMatch(prompt, /\{\{/, "composed prompt must have every placeholder resolved"); + + // b-check (asymmetry baseline): the subagent-directed read clause IS + // delivered to the parent prompt — subagents owe the exact read. + assert.match( + prompt, + /subagents read those `SKILL\.md` files first/, + "parent prompt must carry the subagent-directed SKILL.md read clause", + ); + + // c-check + d-check: the gap. No parent-inline read obligation, and + // the indexed contract body never reaches parent context. + assertParentPromptHasNoInlineSkillReadObligation(prompt); + assertParentPromptOmitsSkillContractBody(prompt, FIXTURE_MARKER); + + console.log( + "inline-skill-probe: fixture indexed (project) -> composed parent prompt has the subagent-only SKILL.md read clause, no parent-inline obligation, no contract body", + ); + } finally { + await runtime.dispose(); + } +} + +if (process.env[CHILD_FLAG] === "1") { + await proveInlineSkillContractGap(); +} else { + test("gentle-shell#348 slice 1: registry-indexed skill contract cannot reach the parent-inline path", () => { + const root = mkdtempSync(join(tmpdir(), "gentle-pi-348-probe-")); + try { + const home = join(root, "home"); + const projectDir = join(root, "project"); + const agentDir = join(home, ".pi", "agent"); + const skillDir = join(projectDir, "skills", FIXTURE_SKILL_NAME); + for (const path of [projectDir, agentDir, skillDir]) { + mkdirSync(path, { recursive: true }); + } + writeFileSync(join(skillDir, "SKILL.md"), FIXTURE_SKILL_SOURCE); + + const shimPath = join(root, "shim", "extensions.ts"); + mkdirSync(dirname(shimPath), { recursive: true }); + writeFileSync(shimPath, buildShimSource()); + + const result = spawnSync( + process.execPath, + ["--experimental-strip-types", fileURLToPath(import.meta.url)], + { + cwd: projectDir, + encoding: "utf8", + timeout: 120_000, + env: { + PATH: `${dirname(process.execPath)}:/usr/bin:/bin`, + HOME: home, + TMPDIR: root, + PI_CODING_AGENT_DIR: agentDir, + GENTLE_PI_AGENT_HOME: agentDir, + GENTLE_PI_CONFIG_HOME: join(home, "config"), + XDG_CONFIG_HOME: join(home, ".config"), + PI_OFFLINE: "1", + GENTLE_PI_348_PROJECT: projectDir, + GENTLE_PI_348_SHIM: shimPath, + [CHILD_FLAG]: "1", + }, + }, + ); + assert.ifError(result.error); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + assert.match(result.stdout, /inline-skill-probe: fixture indexed/); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +} From cb4d4aaf82d71cb4812e20f6a86d7bb98478cc7c Mon Sep 17 00:00:00 2001 From: danielgap Date: Wed, 23 Sep 2026 01:38:53 +0200 Subject: [PATCH 2/4] test(orchestrator): assert the inline-skill gap through the registered before_agent_start handler (#348) CodeRabbit review follow-up on #1357: the repro previously reconstructed the parent prompt via __testing.buildGentlePrompt and asserted against that reconstruction. It now registers the real gentle-ai extension on a minimal capture-pi (accepted tests/runtime-harness.mjs surface), fires the registered before_agent_start handler with the project cwd, and asserts the b/c/d gap checks against the handler-returned systemPrompt, so the repro binds the prompt the primary session actually delivers. Telemetry is deterministically skipped via the GENTLE_AI_TELEMETRY kill switch; the fail-closed unknown RDD line stays pinned. --- ...rchestrator-inline-skill-execution.test.ts | 71 +++++++++++++------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/tests/orchestrator-inline-skill-execution.test.ts b/tests/orchestrator-inline-skill-execution.test.ts index 8cf140a89..c68953ea8 100644 --- a/tests/orchestrator-inline-skill-execution.test.ts +++ b/tests/orchestrator-inline-skill-execution.test.ts @@ -14,12 +14,14 @@ import { fileURLToPath } from "node:url"; // 1. A project skill with an `## Output Contract` body is indexed by the // gentle-ai skill registry (discovery succeeds — `.atl/skill-registry.md` // lists it with scope `project`). -// 2. The composed parent prompt (the exact `buildGentlePrompt` composition the -// primary session's `before_agent_start` handler injects) only binds -// SUBAGENTS to read the exact indexed `SKILL.md` ("subagents read those -// `SKILL.md` files first"). It imposes no parent-inline read obligation: -// there is no "Parent inline execution" section and no wording binding the -// parent's own inline path to read the indexed `SKILL.md` in-session. +// 2. The parent prompt the registered `before_agent_start` handler +// actually returns (persona, cwd, active tool names, RDD status line, +// research capabilities — the real production composition path) only +// binds SUBAGENTS to read the exact indexed `SKILL.md` ("subagents read +// those `SKILL.md` files first"). It imposes no parent-inline read +// obligation: there is no "Parent inline execution" section and no +// wording binding the parent's own inline path to read the indexed +// `SKILL.md` in-session. // 3. Consequently the skill's contract body marker can never reach // parent-inline results through the composed prompt. // @@ -200,22 +202,51 @@ async function proveInlineSkillContractGap(): Promise { "registry index must not inline skill bodies (it carries metadata only)", ); - // Composed parent prompt capture: `buildGentlePrompt` is the exact - // composition the primary session's `before_agent_start` handler - // injects (persona, cwd, active tool names, RDD status line). With - // `nativeReviewCli: null` the resolved RDD line equals the default - // worst-case "unknown" line, so this call reproduces the boot - // composition for the primary session byte-for-byte. The module - // instance is shared with the shim (same absolute path), and - // `getOrchestratorPrompt` memoization is per background-policy key — - // irrelevant here because this repro asserts prompt content, not - // per-cwd variance. - const { __testing } = await import("../extensions/gentle-ai.ts"); + // Parent prompt capture (CodeRabbit review fix): assert against the + // prompt the REGISTERED `before_agent_start` handler returns, not a + // `buildGentlePrompt` reconstruction. The handler is the production + // composition path (persona from ctx.cwd, active tool names via + // pi.getActiveTools, the fail-closed RDD line, research capabilities), + // so the gap assertions below bind the prompt the primary session + // would actually deliver. The capture-pi mirrors the accepted minimal + // registration surface from tests/runtime-harness.mjs; telemetry is + // deterministically skipped through the GENTLE_AI_TELEMETRY kill + // switch, and `nativeReviewCli: null` resolves the worst-case + // "unknown" RDD line exactly like the shim's boot. + const { createGentleAiExtension } = await import("../extensions/gentle-ai.ts"); const toolNames = session.agent.state.tools.map((tool) => tool.name); - const prompt = __testing.buildGentlePrompt("gentleman", cwd, toolNames); + const captureHooks = new Map Promise>>(); + const noop = () => {}; + const capturePi = { + events: { on: () => noop, emit: noop }, + on(name: string, handler: (event: unknown, ctx: unknown) => Promise) { + captureHooks.set(name, [...(captureHooks.get(name) ?? []), handler]); + }, + registerCommand: noop, + registerFlag: noop, + registerTool: noop, + getFlag: () => false, + getCommands: () => [], + getActiveTools: () => toolNames, + getAllTools: () => toolNames.map((name) => ({ name })), + }; + createGentleAiExtension({ + nativeReviewCli: null, + candidateViews: null, + processEnv: { GENTLE_AI_TELEMETRY: "0" }, + })(capturePi as never); + const promptHandler = captureHooks.get("before_agent_start")?.[0]; + assert.ok(promptHandler, "the gentle-ai extension must register a before_agent_start handler"); + const handlerResult = (await promptHandler( + { systemPrompt: "PROBE-BASE" }, + { cwd, hasUI: false, sessionManager: { getSessionId: () => "inline-skill-probe" } }, + )) as { systemPrompt: string }; + const prompt = handlerResult.systemPrompt; - // Fidelity pin: the render actually went through the composition - // pipeline (assets root substitution + background/RDD status block). + // Fidelity pins: the handler result really went through the production + // composition pipeline (base prompt carried through, assets-root + // substitution, background/RDD status block with the fail-closed line). + assert.ok(prompt.startsWith("PROBE-BASE"), "the handler must compose over the base system prompt"); assert.match( prompt, /Receipt-driven development: unknown \(native status unavailable\)/, From e2cf573241fd6d41c306aa27f5aa23bf8217ca20 Mon Sep 17 00:00:00 2001 From: danielgap Date: Fri, 25 Sep 2026 20:34:08 +0200 Subject: [PATCH 3/4] ci(windows): raise candidate-view git timeout for cold runners The Windows candidate view regression fails its first owner-preparation git command with candidate-owner-preparation-failed (ETIMEDOUT) when a cold windows-latest runner exceeds the 10s CANDIDATE_GIT_TIMEOUT_MS cap (#990, #1009 family). Set GENTLE_PI_CANDIDATE_GIT_TIMEOUT_MS=60000 on that step using the override the cap already provides (max 120s) rather than retrying: a git timeout is a diagnosable candidate state, not a transient error. --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1deed4170..84f23223b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,6 +104,8 @@ jobs: - name: Run Windows candidate view regression shell: pwsh + env: + GENTLE_PI_CANDIDATE_GIT_TIMEOUT_MS: "60000" run: | $output = & pnpm exec node --experimental-strip-types --test --test-reporter=tap --test-name-pattern "candidate owner|candidate contributor root|candidate executable-mode validation|candidate registry forwards explicit Windows mode validation" tests/review-candidate-view.test.ts 2>&1 $status = $LASTEXITCODE From 091fe3b4843daf3981b9dfcce26eda8179c75a1b Mon Sep 17 00:00:00 2001 From: danielgap Date: Tue, 29 Sep 2026 01:19:26 +0200 Subject: [PATCH 4/4] test(orchestrator): adapt the #348 probe to the appendSystemPrompt handler contract Main's gentle-shell#1485 changed before_agent_start to compose through the mutable systemPromptOptions.appendSystemPrompt section (pi-claude-bridge drops handler-returned replacement prompts), which broke the probe's handlerResult.systemPrompt capture after the main merge. The probe now passes the structured options and reconstructs the delivered prompt as base plus appended block; the gap assertions are unchanged and the repro still passes on unmodified main. --- ...rchestrator-inline-skill-execution.test.ts | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/orchestrator-inline-skill-execution.test.ts b/tests/orchestrator-inline-skill-execution.test.ts index c68953ea8..99d6cf678 100644 --- a/tests/orchestrator-inline-skill-execution.test.ts +++ b/tests/orchestrator-inline-skill-execution.test.ts @@ -237,16 +237,23 @@ async function proveInlineSkillContractGap(): Promise { })(capturePi as never); const promptHandler = captureHooks.get("before_agent_start")?.[0]; assert.ok(promptHandler, "the gentle-ai extension must register a before_agent_start handler"); - const handlerResult = (await promptHandler( - { systemPrompt: "PROBE-BASE" }, - { cwd, hasUI: false, sessionManager: { getSessionId: () => "inline-skill-probe" } }, - )) as { systemPrompt: string }; - const prompt = handlerResult.systemPrompt; + // gentle-shell#1485: pi-claude-bridge forwards only the structured + // systemPromptOptions and drops a handler-returned replacement prompt, + // so the production handler now composes through the mutable + // appendSystemPrompt section and returns undefined. The probe passes the + // structured options and reconstructs the delivered prompt as the base + // prompt plus the appended block, exactly what the harness would send. + const systemPromptOptions = { appendSystemPrompt: "" }; + await promptHandler( + { systemPrompt: "PROBE-BASE", systemPromptOptions }, + { cwd, hasUI: false, sessionManager: { getSessionId: () => "inline-skill-probe" } }, + ); + const prompt = `PROBE-BASE\n\n${systemPromptOptions.appendSystemPrompt}`; - // Fidelity pins: the handler result really went through the production - // composition pipeline (base prompt carried through, assets-root + // Fidelity pins: the captured composition really went through the + // production pipeline (base prompt carried through, assets-root // substitution, background/RDD status block with the fail-closed line). - assert.ok(prompt.startsWith("PROBE-BASE"), "the handler must compose over the base system prompt"); + assert.ok(prompt.startsWith("PROBE-BASE"), "the probe must compose over the base system prompt"); assert.match( prompt, /Receipt-driven development: unknown \(native status unavailable\)/,