Skip to content

Commit fc0ac9a

Browse files
committed
feat(pi): fan out project scenarios
Run isolated test and service scenarios without rebuilding source in each agent turn.
1 parent 9535db1 commit fc0ac9a

7 files changed

Lines changed: 256 additions & 186 deletions

File tree

packages/pi-extension/README.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,13 @@ bundled files are copied; Pi credentials, settings, and sessions stay local.
6868
The LLM agent always has `sandbox_*` tools to create and manage sandboxes, networks,
6969
disks, port forwarding, and device VPN. Pi's built-in tools stay local unless sandbox mode is enabled.
7070

71-
### Deploy Next.js variants
71+
### Fan out scenarios
7272

73-
For multiple public copies of the current Next.js project, ask Pi once:
73+
For isolated configurations, tests, or deployment checks, ask Pi once:
7474

75-
> Deploy 10 public variants of this current Next.js project. Preserve its tracked source and UI exactly. Render a unique random `HELLO_SUFFIX` in each build, then return only verified public URLs.
75+
> Run these scenarios against independent copies of the current project: unit tests, staging configuration, and production configuration. Preserve the tracked source. Return test results and only health-checked public URLs.
7676
77-
Pi uses `sandbox_deploy_nextjs_variants`: it archives the local project while honoring Git ignore rules, creates sandboxes in bounded parallelism, builds each copy with a distinct suffix, starts each app in `tmux`, and verifies the public HTTPS response. The project must render `HELLO_SUFFIX` during `next build`.
77+
Pi uses `sandbox_fanout`: it archives the local project while honoring Git ignore rules, creates sandboxes in bounded parallelism, and runs each named scenario. A scenario without a port runs a foreground command such as a test suite. A scenario with a port runs its server in `tmux`, then receives a public HTTPS health check. Use scenario environment variables for configuration differences.
7878

7979
### Private networks
8080

packages/pi-extension/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
],
2222
"type": "module",
2323
"scripts": {
24-
"test": "bun test src/startup-sync.test.ts src/nextjs-variants.test.ts",
24+
"test": "bun test src/startup-sync.test.ts src/fanout.test.ts",
2525
"typecheck": "tsc --noEmit"
2626
},
2727
"devDependencies": {
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import assert from "node:assert/strict";
2+
import { test } from "node:test";
3+
4+
import { createHealthCheckUrl, createScenarioCommand } from "./fanout.ts";
5+
6+
test("runs a scenario from the copied project with quoted environment values", () => {
7+
const command = createScenarioCommand({
8+
name: "staging",
9+
command: "bun test",
10+
environment: [
11+
{ name: "API_URL", value: "https://staging.example.test" },
12+
{ name: "FEATURE_FLAG", value: "new flow" },
13+
],
14+
});
15+
16+
assert.match(command, /^cd '\/root\/workspace' && /);
17+
assert.match(command, /API_URL='https:\/\/staging\.example\.test'/);
18+
assert.match(command, /FEATURE_FLAG='new flow' bun test$/);
19+
});
20+
21+
test("keeps health checks on the sandbox ingress origin", () => {
22+
assert.equal(
23+
createHealthCheckUrl("https://sandbox-3000.example.test", "/health").href,
24+
"https://sandbox-3000.example.test/health",
25+
);
26+
assert.throws(
27+
() => createHealthCheckUrl("https://sandbox-3000.example.test", "https://169.254.169.254/"),
28+
/relative/,
29+
);
30+
assert.throws(
31+
() => createHealthCheckUrl("https://sandbox-3000.example.test", "//169.254.169.254/"),
32+
/relative/,
33+
);
34+
});
Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
2+
3+
import * as cli from "./cli.ts";
4+
import { syncProjectOnce } from "./startup-sync.ts";
5+
import { shellQuote } from "./util.ts";
6+
7+
const CONCURRENCY = 3;
8+
const REMOTE_DIR = "/root/workspace";
9+
10+
export interface FanoutScenario {
11+
name: string;
12+
command: string;
13+
environment: Array<{ name: string; value: string }>;
14+
port?: number;
15+
healthCheckPath?: string;
16+
healthCheckContains?: string;
17+
}
18+
19+
export interface FanoutOptions {
20+
sourceDir: string;
21+
namePrefix: string;
22+
scenarios: FanoutScenario[];
23+
shape?: string;
24+
rootfs?: string;
25+
}
26+
27+
export interface FanoutResult {
28+
name: string;
29+
sandboxId?: string;
30+
url?: string;
31+
verified: boolean;
32+
output?: string;
33+
error?: string;
34+
}
35+
36+
export function createScenarioCommand(scenario: FanoutScenario): string {
37+
const environment = scenario.environment.map(({ name, value }) => `${name}=${shellQuote(value)}`).join(" ");
38+
return `cd ${shellQuote(REMOTE_DIR)} && ${environment ? `${environment} ` : ""}${scenario.command}`;
39+
}
40+
41+
export async function fanoutScenarios(
42+
pi: ExtensionAPI,
43+
options: FanoutOptions,
44+
signal?: AbortSignal,
45+
): Promise<FanoutResult[]> {
46+
const results: FanoutResult[] = [];
47+
let nextIndex = 0;
48+
49+
async function worker(): Promise<void> {
50+
while (!signal?.aborted) {
51+
const index = nextIndex++;
52+
if (index >= options.scenarios.length) return;
53+
results[index] = await runScenario(pi, options, options.scenarios[index], signal);
54+
}
55+
}
56+
57+
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, options.scenarios.length) }, worker));
58+
if (signal?.aborted) throw new Error("aborted");
59+
return results;
60+
}
61+
62+
async function runScenario(
63+
pi: ExtensionAPI,
64+
options: FanoutOptions,
65+
scenario: FanoutScenario,
66+
signal?: AbortSignal,
67+
): Promise<FanoutResult> {
68+
const name = `${options.namePrefix}-${scenario.name}`;
69+
let sandboxId: string | undefined;
70+
71+
try {
72+
const sandbox = await cli.createSandbox(pi, {
73+
shape: options.shape,
74+
rootfs: options.rootfs,
75+
name,
76+
ingress: scenario.port !== undefined,
77+
});
78+
sandboxId = sandbox.id;
79+
await syncProjectOnce(pi, sandbox.id, options.sourceDir, {}, signal);
80+
const command = createScenarioCommand(scenario);
81+
82+
if (!scenario.port) return await runForeground(pi, sandbox.id, name, command, signal);
83+
84+
const started = await cli.sandboxExec(
85+
pi,
86+
sandbox.id,
87+
`(tmux kill-session -t scenario 2>/dev/null || true) && tmux new-session -d -s scenario ${shellQuote(command)}`,
88+
signal,
89+
);
90+
if (started.exitCode !== 0) throw new Error(started.stdout || "Scenario start command failed");
91+
92+
const info = await cli.getSandbox(pi, sandbox.id);
93+
const url = info.ingress_url_template?.replace("<port>", String(scenario.port));
94+
if (!url) throw new Error("Ingress URL is unavailable");
95+
96+
const verified = await verifyScenario(url, scenario, signal);
97+
if (!verified) {
98+
await cli.destroySandbox(pi, sandbox.id).catch(() => undefined);
99+
return { name, sandboxId: sandbox.id, verified: false, error: "public health check failed" };
100+
}
101+
return { name, sandboxId: sandbox.id, url, verified: true };
102+
} catch (error) {
103+
if (sandboxId) await cli.destroySandbox(pi, sandboxId).catch(() => undefined);
104+
return {
105+
name,
106+
sandboxId,
107+
verified: false,
108+
error: error instanceof Error ? error.message : String(error),
109+
};
110+
}
111+
}
112+
113+
async function runForeground(
114+
pi: ExtensionAPI,
115+
sandboxId: string,
116+
name: string,
117+
command: string,
118+
signal?: AbortSignal,
119+
): Promise<FanoutResult> {
120+
const result = await cli.sandboxExec(pi, sandboxId, command, signal);
121+
try {
122+
await cli.destroySandbox(pi, sandboxId);
123+
} catch (error) {
124+
return {
125+
name,
126+
sandboxId,
127+
verified: false,
128+
output: result.stdout,
129+
error: `scenario cleanup failed: ${error instanceof Error ? error.message : String(error)}`,
130+
};
131+
}
132+
if (result.exitCode !== 0) {
133+
return { name, sandboxId, verified: false, output: result.stdout, error: "scenario command failed" };
134+
}
135+
return { name, sandboxId, verified: true, output: result.stdout };
136+
}
137+
138+
export function createHealthCheckUrl(url: string, path = "/"): URL {
139+
if (!path.startsWith("/") || path.startsWith("//")) throw new Error("health check path must be relative");
140+
const endpoint = new URL(path, url);
141+
if (endpoint.origin !== new URL(url).origin) throw new Error("health check must use the sandbox ingress URL");
142+
return endpoint;
143+
}
144+
145+
async function verifyScenario(url: string, scenario: FanoutScenario, signal?: AbortSignal): Promise<boolean> {
146+
const endpoint = createHealthCheckUrl(url, scenario.healthCheckPath);
147+
for (let attempt = 0; attempt < 15; attempt += 1) {
148+
if (signal?.aborted) throw new Error("aborted");
149+
try {
150+
const timeout = AbortSignal.timeout(5_000);
151+
const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout;
152+
const response = await fetch(endpoint, { signal: requestSignal, redirect: "error" });
153+
const body = await response.text();
154+
if (response.ok && (!scenario.healthCheckContains || body.includes(scenario.healthCheckContains))) return true;
155+
} catch {
156+
// The process may still be starting.
157+
}
158+
await new Promise((resolve) => setTimeout(resolve, 1_000));
159+
}
160+
return false;
161+
}

packages/pi-extension/src/nextjs-variants.test.ts

Lines changed: 0 additions & 21 deletions
This file was deleted.

packages/pi-extension/src/nextjs-variants.ts

Lines changed: 0 additions & 123 deletions
This file was deleted.

0 commit comments

Comments
 (0)