diff --git a/readme.md b/readme.md index 6d149e9..c502373 100644 --- a/readme.md +++ b/readme.md @@ -574,6 +574,24 @@ orch config edit # Open in $EDITOR +
+Native role workflow + +Run a confirmed, sequential `Supervisor -> Implementer -> Reviewer` workflow. The Adviser is optional and, when selected, is limited to one call. + +```bash +printf '%s' 'Add input validation and tests' | orch workflow --yes \ + --supervisor codex --supervisor-model \ + --implementer claude --implementer-model \ + --reviewer codex --reviewer-model +``` + +Omit `--yes` and use `--objective-file ` for an interactive final confirmation. The summary shows the selected CLIs/models, target branch and SHA, required checks, attempt limits, and the Implementer's autonomous write access inside its dedicated Git worktree before any model CLI starts. + +The workflow currently supports Codex for Supervisor/Reviewer and Claude for Implementer/Adviser. Grok, Antigravity, and other adapters are rejected for this command because a safe prompt transport is not established. The target repository must use one npm lockfile and define `typecheck` and `test` scripts. In the worktree, ORCH runs `npm ci --ignore-scripts`, those checks with npm lifecycle hooks disabled, optional `lint`, and `git diff --check`; any missing, failed, interrupted, or stale result blocks the merge. Immediately before merge, ORCH verifies the original target branch and SHA plus the reviewed worktree, branch, commit, ancestry, and diff hash. + +
+

Aliases: orchestry   orch   ao


diff --git a/src/bin/cli.ts b/src/bin/cli.ts index 622e478..7ae74d5 100644 --- a/src/bin/cli.ts +++ b/src/bin/cli.ts @@ -69,6 +69,7 @@ const COMMAND_STUBS: Array<[name: string, description: string]> = [ ['doctor', 'Check adapters and dependencies'], ['tui', 'Launch TUI dashboard'], ['serve', 'Headless daemon mode with structured logs'], + ['workflow','Run a native role workflow'], ['init', 'Initialize project'], ['update', 'Check for updates'], ]; @@ -106,6 +107,11 @@ async function main(): Promise { } else if (sub === 'update') { const { registerUpdateCommand } = await import('../cli/commands/update.js'); registerUpdateCommand(program); + } else if (sub === 'workflow') { + const { registerWorkflowCommand } = await import('../cli/commands/workflow.js'); + registerWorkflowCommand(program); + await program.parseAsync(process.argv); + return; } // Bare `orch` in a directory without .orchestry/ → auto-init + TUI (FTUE). diff --git a/src/cli/commands/workflow.ts b/src/cli/commands/workflow.ts new file mode 100644 index 0000000..d572905 --- /dev/null +++ b/src/cli/commands/workflow.ts @@ -0,0 +1,78 @@ +import fs from 'node:fs/promises'; +import { createInterface } from 'node:readline/promises'; +import type { Command } from 'commander'; +import { runNativeRoleWorkflow } from '../../infrastructure/workflow/native-role-workflow.js'; + +interface WorkflowOptions { + supervisor: string; + supervisorModel: string; + implementer: string; + implementerModel: string; + reviewer: string; + reviewerModel: string; + adviser?: string; + adviserModel?: string; + objectiveFile?: string; + maxAttempts: string; + yes?: boolean; +} + +export function registerWorkflowCommand(program: Command): void { + program + .command('workflow') + .description('Run a confirmed Supervisor, Implementer, and Reviewer workflow') + .requiredOption('--supervisor ', 'Supervisor CLI (codex)') + .requiredOption('--supervisor-model ', 'Supervisor model') + .requiredOption('--implementer ', 'Implementer CLI (claude)') + .requiredOption('--implementer-model ', 'Implementer model') + .requiredOption('--reviewer ', 'Reviewer CLI (codex)') + .requiredOption('--reviewer-model ', 'Reviewer model') + .option('--adviser ', 'Optional Adviser CLI (claude)') + .option('--adviser-model ', 'Optional Adviser model') + .option('--objective-file ', 'Read the objective from a protected file instead of stdin') + .option('--max-attempts ', 'Maximum Supervisor and Reviewer attempts', '1') + .option('--yes', 'Confirm the printed workflow summary') + .action(async (options: WorkflowOptions) => { + if (Boolean(options.adviser) !== Boolean(options.adviserModel)) { + throw new Error('--adviser and --adviser-model must be supplied together'); + } + if (!options.objectiveFile && !options.yes) { + throw new Error('A workflow objective read from stdin requires --yes; use --objective-file for interactive confirmation'); + } + const objective = options.objectiveFile ? undefined : await readStdin(); + const state = await runNativeRoleWorkflow(process.cwd(), { + objective, + objectiveFile: options.objectiveFile, + confirmed: Boolean(options.yes), + supervisor: { cli: options.supervisor as 'codex', model: options.supervisorModel }, + adviser: options.adviser ? { cli: options.adviser as 'claude', model: options.adviserModel! } : null, + implementer: { cli: options.implementer as 'claude', model: options.implementerModel }, + reviewer: { cli: options.reviewer as 'codex', model: options.reviewerModel }, + maxAttempts: Number(options.maxAttempts), + onSummary: (summary) => console.log(JSON.stringify({ type: 'workflow_summary', ...summary as object })), + confirm: async () => { + const prompt = createInterface({ input: process.stdin, output: process.stdout }); + try { + const answer = (await prompt.question('Start this workflow? [y/N] ')).trim().toLowerCase(); + return answer === 'y' || answer === 'yes'; + } finally { + prompt.close(); + } + }, + onCheck: (check) => console.log(JSON.stringify({ type: 'workflow_check', ...check })), + }); + console.log(JSON.stringify(state)); + }); +} + +async function readStdin(): Promise { + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of process.stdin) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + bytes += buffer.length; + if (bytes > 128_000) throw new Error('Workflow objective exceeds 128000 bytes'); + chunks.push(buffer); + } + return Buffer.concat(chunks).toString('utf8'); +} diff --git a/src/domain/native-role-workflow.ts b/src/domain/native-role-workflow.ts new file mode 100644 index 0000000..59a4019 --- /dev/null +++ b/src/domain/native-role-workflow.ts @@ -0,0 +1,58 @@ +export const WORKFLOW_ROLES = ['supervisor', 'adviser', 'implementer', 'reviewer'] as const; +export type WorkflowRole = typeof WORKFLOW_ROLES[number]; + +export interface WorkflowBinding { + cli: 'codex' | 'claude'; + model: string; +} + +export interface WorkflowAttempt { + id: string; + role: WorkflowRole; + cli: string; + model: string; + status: 'started' | 'succeeded' | 'failed' | 'interrupted'; + started_at: string; + finished_at?: string; + error?: string; +} + +export interface WorkflowCheck { + command: string; + status: 'passed' | 'failed'; + output: string; +} + +export interface NativeRoleWorkflowState { + schema_version: 1; + id: string; + phase: 'created' | 'running' | 'checking' | 'reviewing' | 'merged' | 'cancelled' | 'failed'; + target_branch: string; + target_commit: string; + workflow_branch: string; + worktree: string; + roles: { + supervisor: WorkflowBinding; + adviser: WorkflowBinding | null; + implementer: WorkflowBinding; + reviewer: WorkflowBinding; + }; + attempts: WorkflowAttempt[]; + checks: WorkflowCheck[]; + implementation_commit: string | null; + diff_hash: string | null; + merge_commit: string | null; + error: string | null; +} + +export function validateWorkflowBindings(bindings: NativeRoleWorkflowState['roles']): void { + if (bindings.supervisor.cli !== 'codex') throw new Error('Supervisor must use the Codex CLI'); + if (bindings.implementer.cli !== 'claude') throw new Error('Implementer must use the Claude CLI'); + if (bindings.reviewer.cli !== 'codex') throw new Error('Reviewer must use the Codex CLI'); + if (bindings.adviser && bindings.adviser.cli !== 'claude') throw new Error('Adviser must use the Claude CLI'); + for (const [role, binding] of Object.entries(bindings)) { + if (binding && !/^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$/.test(binding.model)) { + throw new Error(`${role} model is invalid`); + } + } +} diff --git a/src/infrastructure/workflow/native-role-workflow.ts b/src/infrastructure/workflow/native-role-workflow.ts new file mode 100644 index 0000000..1cdc548 --- /dev/null +++ b/src/infrastructure/workflow/native-role-workflow.ts @@ -0,0 +1,508 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { execFile, spawn } from 'node:child_process'; +import fs from 'node:fs/promises'; +import { constants as fsConstants } from 'node:fs'; +import path from 'node:path'; +import { promisify } from 'node:util'; +import type { + NativeRoleWorkflowState, + WorkflowBinding, + WorkflowRole, +} from '../../domain/native-role-workflow.js'; +import { validateWorkflowBindings } from '../../domain/native-role-workflow.js'; + +const execFileAsync = promisify(execFile); +const MAX_OBJECTIVE_BYTES = 128_000; +const MAX_PROCESS_OUTPUT_BYTES = 1_000_000; + +export interface NativeRoleWorkflowOptions { + objective?: string; + objectiveFile?: string; + confirmed: boolean; + supervisor: WorkflowBinding; + adviser: WorkflowBinding | null; + implementer: WorkflowBinding; + reviewer: WorkflowBinding; + maxAttempts: number; + onSummary?: (summary: unknown) => void; + confirm?: () => Promise; + onCheck?: (check: { command: string; status: string }) => void; +} + +interface RepositorySnapshot { + root: string; + targetBranch: string; + targetCommit: string; + packageManager: 'npm'; + scripts: Array<{ name: string; value: string }>; + lockfile: string; +} + +export async function runNativeRoleWorkflow( + cwd: string, + options: NativeRoleWorkflowOptions, +): Promise { + const objective = await readObjective(options); + const roles = { + supervisor: options.supervisor, + adviser: options.adviser, + implementer: options.implementer, + reviewer: options.reviewer, + }; + validateWorkflowBindings(roles); + if (!Number.isSafeInteger(options.maxAttempts) || options.maxAttempts < 1 || options.maxAttempts > 3) { + throw new Error('max-attempts must be between 1 and 3'); + } + + const repository = await inspectRepository(cwd); + await assertExecutables(roles); + const id = `wf_${randomUUID().replaceAll('-', '').slice(0, 12)}`; + const workflowRoot = path.join(repository.root, '.orchestry', 'workflows', id); + const worktree = path.join(workflowRoot, 'worktree'); + const workflowBranch = `orchestry/workflow-${id.slice(3)}`; + const state: NativeRoleWorkflowState = { + schema_version: 1, + id, + phase: 'created', + target_branch: repository.targetBranch, + target_commit: repository.targetCommit, + workflow_branch: workflowBranch, + worktree, + roles, + attempts: [], + checks: [], + implementation_commit: null, + diff_hash: null, + merge_commit: null, + error: null, + }; + await fs.mkdir(workflowRoot, { recursive: true, mode: 0o700 }); + await writeState(workflowRoot, state); + + options.onSummary?.({ + workflow_id: id, + objective: { supplied: true, bytes: Buffer.byteLength(objective) }, + target: { branch: repository.targetBranch, commit: repository.targetCommit }, + roles, + checks: ['npm ci --ignore-scripts --no-audit --no-fund', ...repository.scripts.map((script) => `npm run --ignore-scripts ${script.name}`), 'git diff --check'], + adviser: { enabled: roles.adviser !== null, max_calls: roles.adviser ? 1 : 0 }, + implementer_permissions: 'autonomous writes limited to the dedicated Git worktree', + max_attempts: { supervisor: options.maxAttempts, adviser: roles.adviser ? 1 : 0, implementer: 1, reviewer: options.maxAttempts }, + }); + + const accepted = options.confirmed || await options.confirm?.() || false; + if (!accepted) { + state.phase = 'cancelled'; + await writeState(workflowRoot, state); + return state; + } + + let worktreeCreated = false; + try { + await assertTargetUnchanged(repository); + await git(repository.root, ['worktree', 'add', '-b', workflowBranch, worktree, repository.targetCommit]); + worktreeCreated = true; + state.phase = 'running'; + await writeState(workflowRoot, state); + + const supervisor = await runRoleWithRetries({ + role: 'supervisor', binding: roles.supervisor, cwd: repository.root, + prompt: JSON.stringify({ role: 'supervisor', objective, task: 'Return JSON with a nonempty plan string.' }), + attempts: options.maxAttempts, state, workflowRoot, + }); + const plan = supervisor['plan'] as string; + + let advice: string | null = null; + if (roles.adviser) { + const adviser = await runRoleWithRetries({ + role: 'adviser', binding: roles.adviser, cwd: repository.root, + prompt: JSON.stringify({ role: 'adviser', objective, plan, task: 'Return JSON with an advice string. Do not modify files.' }), + attempts: 1, state, workflowRoot, + }); + advice = adviser['advice'] as string; + } + + const implementerPrompt = JSON.stringify({ + role: 'implementer', objective, plan, advice, + task: 'Implement the objective in this Git worktree, run no network commands, and commit all changes. Return JSON with status "completed".', + }); + const implementation = await runRoleWithRetries({ + role: 'implementer', binding: roles.implementer, cwd: worktree, + prompt: implementerPrompt, attempts: 1, state, workflowRoot, + }); + if (implementation['status'] !== 'completed') throw new Error('Implementer did not report completed status'); + + await assertClean(worktree, 'Implementation worktree'); + state.implementation_commit = await gitOutput(worktree, ['rev-parse', 'HEAD']); + if (state.implementation_commit === repository.targetCommit) throw new Error('Implementer produced no commit'); + if (!await gitSucceeds(worktree, ['merge-base', '--is-ancestor', repository.targetCommit, state.implementation_commit])) { + throw new Error('Implementation commit does not descend from the recorded target'); + } + await assertFrozenManifest(worktree, repository); + const diff = await gitOutput(worktree, ['diff', '--binary', `${repository.targetCommit}..${state.implementation_commit}`]); + if (!diff.trim()) throw new Error('Implementer produced an empty diff'); + state.diff_hash = createHash('sha256').update(diff).digest('hex'); + + state.phase = 'checking'; + await writeState(workflowRoot, state); + const installCommand = 'npm ci --ignore-scripts --no-audit --no-fund'; + const install = await runCheck('npm', ['ci', '--ignore-scripts', '--no-audit', '--no-fund'], worktree, installCommand); + state.checks.push(install); + options.onCheck?.({ command: install.command, status: install.status }); + await writeState(workflowRoot, state); + if (install.status !== 'passed') throw new Error(`Required check failed: ${installCommand}`); + for (const script of repository.scripts) { + const command = `npm run --ignore-scripts ${script.name}`; + const check = await runCheck('npm', ['run', '--ignore-scripts', script.name], worktree, command); + state.checks.push(check); + options.onCheck?.({ command, status: check.status }); + await writeState(workflowRoot, state); + if (check.status !== 'passed') throw new Error(`Required check failed: ${command}`); + } + const diffCheck = await runCheck('git', ['diff', '--check', `${repository.targetCommit}..${state.implementation_commit}`], worktree, 'git diff --check'); + state.checks.push(diffCheck); + options.onCheck?.({ command: diffCheck.command, status: diffCheck.status }); + await writeState(workflowRoot, state); + if (diffCheck.status !== 'passed') throw new Error('Required check failed: git diff --check'); + + const checkedCommit = await gitOutput(worktree, ['rev-parse', 'HEAD']); + const checkedDiff = await gitOutput(worktree, ['diff', '--binary', `${repository.targetCommit}..${checkedCommit}`]); + if (checkedCommit !== state.implementation_commit || createHash('sha256').update(checkedDiff).digest('hex') !== state.diff_hash) { + throw new Error('Implementation changed while checks were running'); + } + + state.phase = 'reviewing'; + await writeState(workflowRoot, state); + const review = await runRoleWithRetries({ + role: 'reviewer', binding: roles.reviewer, cwd: worktree, + prompt: JSON.stringify({ + role: 'reviewer', objective, plan, advice, + implementation_commit: state.implementation_commit, + diff_hash: state.diff_hash, + diff: checkedDiff, + checks: state.checks.map(({ command, status }) => ({ command, status })), + task: 'Return JSON with decision "accept" or "reject" and a reason string. Do not modify files.', + }), + attempts: options.maxAttempts, state, workflowRoot, + }); + if (review['decision'] !== 'accept') throw new Error(`Reviewer rejected the implementation: ${String(review['reason'] ?? 'no reason')}`); + + await assertMergeSafety(repository, state); + await assertTargetUnchanged(repository); + await mergeReviewedCommit(repository.root, state.implementation_commit); + state.merge_commit = await gitOutput(repository.root, ['rev-parse', 'HEAD']); + if (state.merge_commit === repository.targetCommit || !await gitSucceeds(repository.root, ['merge-base', '--is-ancestor', state.implementation_commit, state.merge_commit])) { + throw new Error('Merge did not incorporate the reviewed implementation commit'); + } + state.phase = 'merged'; + await writeState(workflowRoot, state); + return state; + } catch (error) { + state.phase = 'failed'; + state.error = error instanceof Error ? error.message : String(error); + await writeState(workflowRoot, state); + throw Object.assign(new Error(state.error), { workflowState: state }); + } finally { + if (worktreeCreated) { + try { + await git(repository.root, ['worktree', 'remove', '--force', worktree]); + await git(repository.root, ['branch', state.phase === 'merged' ? '-d' : '-D', workflowBranch]); + } catch (cleanupError) { + const completedMerge = state.phase === 'merged'; + const detail = cleanupError instanceof Error ? cleanupError.message : String(cleanupError); + state.error = state.error ? `${state.error}; cleanup failed: ${detail}` : `Workflow cleanup failed: ${detail}`; + if (completedMerge) state.phase = 'failed'; + await writeState(workflowRoot, state); + if (completedMerge) throw new Error(state.error); + } + } + } +} + +async function readObjective(options: NativeRoleWorkflowOptions): Promise { + if (Boolean(options.objective) === Boolean(options.objectiveFile)) { + throw new Error('Provide the objective through stdin or --objective-file, but not both'); + } + let value: string; + if (options.objectiveFile) { + const resolved = path.resolve(options.objectiveFile); + const stat = await fs.lstat(resolved); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_OBJECTIVE_BYTES) { + throw new Error('Objective file must be a regular non-symlink file within the size limit'); + } + const handle = await fs.open(resolved, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + try { + const opened = await handle.stat(); + if (!opened.isFile() || opened.dev !== stat.dev || opened.ino !== stat.ino || opened.size !== stat.size) { + throw new Error('Objective file changed during validation'); + } + value = await handle.readFile('utf8'); + } finally { + await handle.close(); + } + } else { + value = options.objective ?? ''; + } + if (Buffer.byteLength(value) > MAX_OBJECTIVE_BYTES) throw new Error('Workflow objective exceeds 128000 bytes'); + const objective = value.trim(); + if (!objective) throw new Error('Workflow objective must not be empty'); + return objective; +} + +async function inspectRepository(cwd: string): Promise { + const root = await gitOutput(cwd, ['rev-parse', '--show-toplevel']); + const targetBranch = await gitOutput(root, ['symbolic-ref', '--short', 'HEAD']); + const targetCommit = await gitOutput(root, ['rev-parse', 'HEAD']); + await assertClean(root, 'Target repository'); + const manifestPath = path.join(root, 'package.json'); + const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as { scripts?: Record }; + const scripts = ['typecheck', 'test', 'lint'].flatMap((name) => { + const value = manifest.scripts?.[name]; + return typeof value === 'string' && value.trim() ? [{ name, value }] : []; + }); + if (!scripts.some((script) => script.name === 'typecheck') || !scripts.some((script) => script.name === 'test')) { + throw new Error('Target repository must define meaningful typecheck and test package scripts'); + } + const lockfiles = ['package-lock.json', 'npm-shrinkwrap.json']; + const present = (await Promise.all(lockfiles.map(async (name) => fs.access(path.join(root, name)).then(() => name, () => null)))).filter((name): name is string => Boolean(name)); + if (present.length !== 1) throw new Error('Target repository must contain exactly one npm lockfile'); + return { root, targetBranch, targetCommit, packageManager: 'npm', scripts, lockfile: present[0]! }; +} + +async function assertExecutables(roles: NativeRoleWorkflowState['roles']): Promise { + for (const cli of new Set(Object.values(roles).filter(Boolean).map((binding) => binding!.cli))) { + const found = await findExecutable(cli); + if (!found) throw new Error(`${cli} CLI is not available on PATH`); + } +} + +async function findExecutable(command: string): Promise { + for (const directory of (process.env['PATH'] ?? '').split(path.delimiter)) { + if (!directory) continue; + const candidate = path.join(directory, command); + if (await fs.access(candidate, fsConstants.X_OK).then(() => true, () => false)) return candidate; + } + return null; +} + +async function runRoleWithRetries(input: { + role: WorkflowRole; + binding: WorkflowBinding; + cwd: string; + prompt: string; + attempts: number; + state: NativeRoleWorkflowState; + workflowRoot: string; +}): Promise> { + let lastError: Error | null = null; + for (let number = 1; number <= input.attempts; number++) { + const attempt = { + id: `${input.role}-${number}-${randomUUID().slice(0, 8)}`, + role: input.role, + cli: input.binding.cli, + model: input.binding.model, + status: 'started' as const, + started_at: new Date().toISOString(), + }; + input.state.attempts.push(attempt); + await writeState(input.workflowRoot, input.state); + try { + const value = await spawnRole(input.role, input.binding, input.cwd, input.prompt); + validateRoleResult(input.role, value); + Object.assign(attempt, { status: 'succeeded', finished_at: new Date().toISOString() }); + await writeState(input.workflowRoot, input.state); + return value; + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + Object.assign(attempt, { status: 'failed', finished_at: new Date().toISOString(), error: lastError.message }); + await writeState(input.workflowRoot, input.state); + } + } + throw lastError ?? new Error(`${input.role} failed`); +} + +async function spawnRole(role: WorkflowRole, binding: WorkflowBinding, cwd: string, prompt: string): Promise> { + const args = binding.cli === 'codex' + ? ['exec', '--json', '--sandbox', 'read-only', '--model', binding.model, '-'] + : [ + '--print', '--output-format', 'stream-json', '--verbose', '--model', binding.model, + ...(role === 'implementer' ? ['--dangerously-skip-permissions'] : []), + ...(role === 'adviser' + ? ['--max-turns', '1', '--bare', '--tools', '', '--disable-slash-commands', '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}', '--no-session-persistence'] + : []), + ]; + const env = childEnvironment(role, cwd); + const output = await new Promise((resolve, reject) => { + const child = spawn(binding.cli, args, { cwd, env, stdio: ['pipe', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + let settled = false; + let timedOut = false; + let killTimer: ReturnType | undefined; + const timer = setTimeout(() => { + if (settled) return; + timedOut = true; + child.kill('SIGTERM'); + killTimer = setTimeout(() => { + if (!settled) child.kill('SIGKILL'); + }, 2_000); + }, 600_000); + const append = (current: string, chunk: Buffer) => (current + chunk.toString()).slice(-MAX_PROCESS_OUTPUT_BYTES); + child.stdout.on('data', (chunk: Buffer) => { stdout = append(stdout, chunk); }); + child.stderr.on('data', (chunk: Buffer) => { stderr = append(stderr, chunk); }); + child.on('error', (error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (killTimer) clearTimeout(killTimer); + reject(error); + }); + child.on('close', (code) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (killTimer) clearTimeout(killTimer); + if (timedOut) reject(new Error(`${role} timed out`)); + else if (code !== 0) reject(new Error(`${role} CLI exited with code ${code}`)); + else resolve(stdout); + }); + child.stdin.end(prompt); + }); + let resultText = ''; + const lines = output.trim().split('\n').filter(Boolean); + for (const line of lines) { + try { + const parsed: unknown = JSON.parse(line); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue; + const event = parsed as Record; + if (binding.cli === 'codex') { + const item = event['item']; + if (item && typeof item === 'object' && !Array.isArray(item)) { + const message = item as Record; + if (message['type'] === 'agent_message' && typeof message['text'] === 'string') resultText = message['text']; + } + } else if (event['type'] === 'result' && typeof event['result'] === 'string') { + resultText = event['result']; + } else if (!('type' in event)) { + resultText = line; + } + } catch { /* Continue to the previous JSONL record. */ } + } + if (resultText) { + try { + const parsed: unknown = JSON.parse(resultText); + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) return parsed as Record; + } catch { /* Report malformed structured output below. */ } + } + throw new Error(`${role} returned malformed JSON`); +} + +function childEnvironment(role: WorkflowRole, cwd: string): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + PATH: process.env['PATH'], + HOME: process.env['HOME'], + TMPDIR: process.env['TMPDIR'], + LANG: process.env['LANG'], + HTTP_PROXY: process.env['HTTP_PROXY'], + HTTPS_PROXY: process.env['HTTPS_PROXY'], + NO_PROXY: process.env['NO_PROXY'], + http_proxy: process.env['http_proxy'], + https_proxy: process.env['https_proxy'], + no_proxy: process.env['no_proxy'], + ORCH_WORKFLOW_ROLE: role, + }; + return Object.fromEntries(Object.entries(env).filter((entry): entry is [string, string] => entry[1] !== undefined)); +} + +async function runCheck(command: string, args: string[], cwd: string, display: string) { + try { + const { stdout, stderr } = await execFileAsync(command, args, { cwd, maxBuffer: MAX_PROCESS_OUTPUT_BYTES }); + void stdout; + void stderr; + return { command: display, status: 'passed' as const, output: 'Command completed successfully' }; + } catch { + return { command: display, status: 'failed' as const, output: 'Command exited non-zero' }; + } +} + +async function assertFrozenManifest(worktree: string, repository: RepositorySnapshot): Promise { + const manifest = JSON.parse(await fs.readFile(path.join(worktree, 'package.json'), 'utf8')) as { scripts?: Record }; + for (const script of repository.scripts) { + if (manifest.scripts?.[script.name] !== script.value) throw new Error(`Implementer changed the ${script.name} check definition`); + } + await fs.access(path.join(worktree, repository.lockfile)); +} + +async function assertTargetUnchanged(repository: RepositorySnapshot): Promise { + const branch = await gitOutput(repository.root, ['symbolic-ref', '--short', 'HEAD']); + const commit = await gitOutput(repository.root, ['rev-parse', 'HEAD']); + if (branch !== repository.targetBranch || commit !== repository.targetCommit) throw new Error('Target branch changed before workflow execution'); + await assertClean(repository.root, 'Target repository'); +} + +async function assertMergeSafety(repository: RepositorySnapshot, state: NativeRoleWorkflowState): Promise { + if (!state.implementation_commit || !state.diff_hash) throw new Error('Reviewed implementation evidence is incomplete'); + await assertClean(state.worktree, 'Implementation worktree'); + const worktreeRoot = await gitOutput(state.worktree, ['rev-parse', '--show-toplevel']); + const worktreeBranch = await gitOutput(state.worktree, ['symbolic-ref', '--short', 'HEAD']); + if (path.resolve(worktreeRoot) !== path.resolve(state.worktree) || worktreeBranch !== state.workflow_branch) { + throw new Error('Implementation worktree no longer matches the workflow job'); + } + const branchCommit = await gitOutput(state.worktree, ['rev-parse', 'HEAD']); + const diff = await gitOutput(state.worktree, ['diff', '--binary', `${repository.targetCommit}..${branchCommit}`]); + const hash = createHash('sha256').update(diff).digest('hex'); + if (branchCommit !== state.implementation_commit || hash !== state.diff_hash) throw new Error('Reviewed implementation changed before merge'); +} + +async function mergeReviewedCommit(root: string, commit: string): Promise { + try { + await git(root, ['merge', '--no-ff', '--no-edit', commit]); + } catch (error) { + const mergeHead = await gitOutput(root, ['rev-parse', '--git-path', 'MERGE_HEAD']); + if (await fs.access(mergeHead).then(() => true, () => false)) { + await git(root, ['merge', '--abort']); + } + await assertClean(root, 'Target repository after failed merge'); + throw error; + } +} + +async function assertClean(cwd: string, label: string): Promise { + const status = await gitOutput(cwd, ['status', '--porcelain']); + if (status) throw new Error(`${label} must be clean`); +} + +async function writeState(root: string, state: NativeRoleWorkflowState): Promise { + const target = path.join(root, 'state.json'); + const temporary = `${target}.${process.pid}.tmp`; + await fs.writeFile(temporary, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 }); + await fs.rename(temporary, target); +} + +function validateRoleResult(role: WorkflowRole, value: Record): void { + if (role === 'supervisor' && (typeof value['plan'] !== 'string' || !value['plan'].trim())) { + throw new Error('Supervisor returned no plan'); + } + if (role === 'adviser' && (typeof value['advice'] !== 'string' || !value['advice'].trim())) { + throw new Error('Adviser returned no advice'); + } + if (role === 'implementer' && value['status'] !== 'completed') { + throw new Error('Implementer did not report completed status'); + } + if (role === 'reviewer' && value['decision'] !== 'accept' && value['decision'] !== 'reject') { + throw new Error('Reviewer returned no valid decision'); + } +} + +async function git(cwd: string, args: string[]): Promise { + await execFileAsync('git', args, { cwd, maxBuffer: MAX_PROCESS_OUTPUT_BYTES }); +} + +async function gitSucceeds(cwd: string, args: string[]): Promise { + return execFileAsync('git', args, { cwd, maxBuffer: MAX_PROCESS_OUTPUT_BYTES }).then(() => true, () => false); +} + +async function gitOutput(cwd: string, args: string[]): Promise { + const { stdout } = await execFileAsync('git', args, { cwd, maxBuffer: MAX_PROCESS_OUTPUT_BYTES }); + return stdout.trim(); +} diff --git a/test/fixtures/fake-native-role-cli.mjs b/test/fixtures/fake-native-role-cli.mjs new file mode 100644 index 0000000..39ee2df --- /dev/null +++ b/test/fixtures/fake-native-role-cli.mjs @@ -0,0 +1,78 @@ +#!/usr/bin/env node +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; + +const command = path.basename(process.argv[1]); +const argv = process.argv.slice(2); +const input = fs.readFileSync(0, 'utf8'); +const home = process.env.HOME; +if (!home) process.exit(90); +fs.appendFileSync(path.join(home, 'native-role-calls.jsonl'), `${JSON.stringify({ + command, + argv, + cwd: process.cwd(), + stdin: input, + role: process.env.ORCH_WORKFLOW_ROLE, + proxy: process.env.HTTPS_PROXY, +})}\n`); + +let request; +try { + request = JSON.parse(input); +} catch { + console.error('Prompt was not valid JSON'); + process.exit(91); +} + +const objective = String(request.objective ?? ''); +const role = process.env.ORCH_WORKFLOW_ROLE; +if (role === 'supervisor') { + if (objective.includes('SUPERVISOR_RETRY')) { + const calls = fs.readFileSync(path.join(home, 'native-role-calls.jsonl'), 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + if (calls.filter((call) => call.role === 'supervisor').length === 1) { + console.log(JSON.stringify({ type: 'item.completed', item: { type: 'agent_message', text: '{}' } })); + process.exit(0); + } + } + console.log(JSON.stringify({ type: 'item.completed', item: { type: 'agent_message', text: JSON.stringify({ plan: 'Implement the requested deterministic fixture change.' }) } })); + console.log(JSON.stringify({ type: 'turn.completed' })); + process.exit(0); +} +if (role === 'adviser') { + console.log(JSON.stringify({ type: 'result', result: JSON.stringify({ advice: 'Keep the fixture change minimal.' }) })); + process.exit(0); +} +if (role === 'implementer') { + fs.writeFileSync('result.txt', `${objective}\n`); + if (objective.includes('FAIL_CHECK')) fs.writeFileSync('fail-check', 'fail\n'); + execFileSync('git', ['add', 'result.txt', ...(objective.includes('FAIL_CHECK') ? ['fail-check'] : [])]); + if (objective.includes('NON_DESCENDANT')) { + const tree = execFileSync('git', ['write-tree'], { encoding: 'utf8' }).trim(); + const commit = execFileSync('git', ['commit-tree', tree, '-m', 'unrelated implementation'], { encoding: 'utf8' }).trim(); + execFileSync('git', ['reset', '--hard', commit]); + } else { + execFileSync('git', ['commit', '-m', 'implement fixture objective']); + } + console.log(JSON.stringify({ type: 'result', result: JSON.stringify({ status: 'completed' }) })); + process.exit(0); +} +if (role === 'reviewer') { + if (objective.includes('REVIEW_FAIL')) { + console.error('deterministic reviewer failure'); + process.exit(17); + } + if (objective.includes('TARGET_ADVANCE')) { + const common = execFileSync('git', ['rev-parse', '--git-common-dir'], { encoding: 'utf8' }).trim(); + const root = path.dirname(path.resolve(process.cwd(), common)); + execFileSync('git', ['commit', '--allow-empty', '-m', 'concurrent target advance'], { cwd: root }); + } + if (objective.includes('REVIEW_REJECT')) { + console.log(JSON.stringify({ type: 'item.completed', item: { type: 'agent_message', text: JSON.stringify({ decision: 'reject', reason: 'Deterministic rejection.' }) } })); + process.exit(0); + } + console.log(JSON.stringify({ type: 'item.completed', item: { type: 'agent_message', text: JSON.stringify({ decision: 'accept', reason: 'Deterministic review passed.' }) } })); + console.log(JSON.stringify({ type: 'turn.completed' })); + process.exit(0); +} +process.exit(93); diff --git a/test/integration/native-role-workflow.test.ts b/test/integration/native-role-workflow.test.ts new file mode 100644 index 0000000..817059e --- /dev/null +++ b/test/integration/native-role-workflow.test.ts @@ -0,0 +1,253 @@ +import { execFile } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { promisify } from 'node:util'; +import { afterEach, describe, expect, it } from 'vitest'; + +const execFileAsync = promisify(execFile); +const roots: string[] = []; +const projectRoot = path.resolve('.'); +const tsx = path.join(projectRoot, 'node_modules', '.bin', 'tsx'); +const cli = path.join(projectRoot, 'src', 'bin', 'cli.ts'); + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); +}); + +describe('native role workflow CLI', () => { + it('runs the direct path with real subprocesses, commits, checks, merge, and worktree cleanup', async () => { + const fixture = await createFixture(); + const result = await runWorkflow(fixture, 'SUCCESS_OBJECTIVE'); + + expect(result.code).toBe(0); + const state = await latestState(fixture.repo); + expect(state.phase).toBe('merged'); + expect(state.attempts.map((attempt) => `${attempt.role}:${attempt.status}`)).toEqual([ + 'supervisor:succeeded', + 'implementer:succeeded', + 'reviewer:succeeded', + ]); + expect(state.checks.map((check) => `${check.command}:${check.status}`)).toEqual([ + 'npm ci --ignore-scripts --no-audit --no-fund:passed', + 'npm run --ignore-scripts typecheck:passed', + 'npm run --ignore-scripts test:passed', + 'git diff --check:passed', + ]); + expect(await fs.readFile(path.join(fixture.repo, 'result.txt'), 'utf8')).toBe('SUCCESS_OBJECTIVE\n'); + expect(await exists(state.worktree)).toBe(false); + expect(await git(fixture.repo, ['branch', '--list', state.workflow_branch])).toBe(''); + }); + + it('bounds an enabled Adviser to one successful subprocess', async () => { + const fixture = await createFixture(); + const result = await runWorkflow(fixture, 'ADVISER_OBJECTIVE', ['--adviser', 'claude', '--adviser-model', 'adviser-model']); + + expect(result.code).toBe(0); + const state = await latestState(fixture.repo); + expect(state.attempts.filter((attempt) => attempt.role === 'adviser').map((attempt) => attempt.status)).toEqual(['succeeded']); + const adviserCalls = (await calls(fixture.home)).filter((call) => call.role === 'adviser'); + expect(adviserCalls).toHaveLength(1); + expect(adviserCalls[0]?.argv).toContain('--no-session-persistence'); + }); + + it('blocks merge when a required repository check fails', async () => { + const fixture = await createFixture(); + const before = await git(fixture.repo, ['rev-parse', 'HEAD']); + const result = await runWorkflow(fixture, 'FAIL_CHECK'); + + expect(result.code).toBe(1); + const state = await latestState(fixture.repo); + expect(state.phase).toBe('failed'); + expect(state.error).toContain('Required check failed: npm run --ignore-scripts test'); + expect(await git(fixture.repo, ['rev-parse', 'HEAD'])).toBe(before); + expect(await exists(path.join(fixture.repo, 'result.txt'))).toBe(false); + }); + + it('rejects an implementation commit outside the recorded target history', async () => { + const fixture = await createFixture(); + const before = await git(fixture.repo, ['rev-parse', 'HEAD']); + const result = await runWorkflow(fixture, 'NON_DESCENDANT'); + + expect(result.code).toBe(1); + const state = await latestState(fixture.repo); + expect(state.error).toContain('does not descend from the recorded target'); + expect(await git(fixture.repo, ['rev-parse', 'HEAD'])).toBe(before); + }); + + it('refuses merge when the recorded target branch advances', async () => { + const fixture = await createFixture(); + const before = await git(fixture.repo, ['rev-parse', 'HEAD']); + const result = await runWorkflow(fixture, 'TARGET_ADVANCE'); + + expect(result.code).toBe(1); + const state = await latestState(fixture.repo); + expect(state.phase).toBe('failed'); + expect(state.error).toContain('Target branch changed'); + expect(await git(fixture.repo, ['rev-parse', 'HEAD'])).not.toBe(before); + expect(await exists(path.join(fixture.repo, 'result.txt'))).toBe(false); + }); + + it('accounts for reviewer failures exactly without retrying past the bound', async () => { + const fixture = await createFixture(); + const result = await runWorkflow(fixture, 'REVIEW_FAIL'); + + expect(result.code).toBe(1); + const state = await latestState(fixture.repo); + expect(state.attempts.map((attempt) => `${attempt.role}:${attempt.status}`)).toEqual([ + 'supervisor:succeeded', + 'implementer:succeeded', + 'reviewer:failed', + ]); + expect((await calls(fixture.home)).filter((call) => call.role === 'reviewer')).toHaveLength(1); + }); + + it('counts invalid structured output as failed before a bounded retry', async () => { + const fixture = await createFixture(); + const result = await runWorkflow(fixture, 'SUPERVISOR_RETRY', ['--max-attempts', '2']); + + expect(result.code).toBe(0); + const state = await latestState(fixture.repo); + expect(state.attempts.filter((attempt) => attempt.role === 'supervisor').map((attempt) => attempt.status)).toEqual(['failed', 'succeeded']); + }); + + it('does not merge a Reviewer rejection', async () => { + const fixture = await createFixture(); + const before = await git(fixture.repo, ['rev-parse', 'HEAD']); + const result = await runWorkflow(fixture, 'REVIEW_REJECT'); + + expect(result.code).toBe(1); + const state = await latestState(fixture.repo); + expect(state.error).toContain('Reviewer rejected the implementation'); + expect(await git(fixture.repo, ['rev-parse', 'HEAD'])).toBe(before); + }); + + it('stops before any role subprocess when confirmation is refused', async () => { + const fixture = await createFixture(); + const objectiveFile = path.join(fixture.root, 'objective.txt'); + await fs.writeFile(objectiveFile, 'REFUSED_OBJECTIVE\n', { mode: 0o600 }); + const result = await runWorkflow(fixture, 'n\n', ['--objective-file', objectiveFile], false); + + expect(result.code).toBe(0); + expect((await latestState(fixture.repo)).phase).toBe('cancelled'); + expect(await calls(fixture.home)).toEqual([]); + }); + + it('keeps objectives and role prompts out of every subprocess argv', async () => { + const fixture = await createFixture(); + const objective = 'ARGV_SECRET_SENTINEL'; + const result = await runWorkflow(fixture, objective); + + expect(result.code).toBe(0); + const observed = await calls(fixture.home); + expect(observed).toHaveLength(3); + expect(observed.every((call) => !call.argv.join(' ').includes(objective))).toBe(true); + expect(observed.every((call) => call.stdin.includes(objective))).toBe(true); + expect(observed.every((call) => call.proxy === 'http://127.0.0.1:9')).toBe(true); + const implementer = observed.find((call) => call.role === 'implementer'); + expect(implementer?.argv).toContain('--dangerously-skip-permissions'); + const adviser = observed.find((call) => call.role === 'adviser'); + expect(adviser).toBeUndefined(); + }); +}); + +interface Fixture { + root: string; + repo: string; + home: string; + bin: string; +} + +async function createFixture(): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'orch-native-role-')); + roots.push(root); + const repo = path.join(root, 'repo'); + const home = path.join(root, 'home'); + const bin = path.join(root, 'bin'); + await Promise.all([fs.mkdir(repo), fs.mkdir(home), fs.mkdir(bin)]); + for (const command of ['codex', 'claude']) { + const target = path.join(bin, command); + await fs.copyFile(path.join(projectRoot, 'test', 'fixtures', 'fake-native-role-cli.mjs'), target); + await fs.chmod(target, 0o755); + } + await fs.writeFile(path.join(repo, 'package.json'), JSON.stringify({ + name: 'workflow-fixture', + version: '1.0.0', + scripts: { + typecheck: 'node check.mjs typecheck', + test: 'node check.mjs test', + }, + }, null, 2)); + await fs.writeFile(path.join(repo, 'package-lock.json'), JSON.stringify({ + name: 'workflow-fixture', + version: '1.0.0', + lockfileVersion: 3, + requires: true, + packages: { '': { name: 'workflow-fixture', version: '1.0.0' } }, + }, null, 2)); + await fs.writeFile(path.join(repo, 'check.mjs'), "import fs from 'node:fs'; if (process.argv[2] === 'test' && fs.existsSync('fail-check')) process.exit(1);\n"); + await fs.writeFile(path.join(repo, '.gitignore'), '.orchestry/\n'); + await git(repo, ['init', '-b', 'main']); + await git(repo, ['config', 'user.name', 'ORCH Test']); + await git(repo, ['config', 'user.email', 'orch-test@example.invalid']); + await git(repo, ['add', '.']); + await git(repo, ['commit', '-m', 'fixture baseline']); + return { root, repo, home, bin }; +} + +async function runWorkflow(fixture: Fixture, input: string, extra: string[] = [], yes = true) { + const args = [cli, 'workflow', + '--supervisor', 'codex', '--supervisor-model', 'supervisor-model', + '--implementer', 'claude', '--implementer-model', 'implementer-model', + '--reviewer', 'codex', '--reviewer-model', 'reviewer-model', + '--max-attempts', '1', + ...(yes ? ['--yes'] : []), + ...extra, + ]; + return new Promise<{ code: number; stdout: string; stderr: string }>((resolve) => { + const child = execFile(tsx, args, { + cwd: fixture.repo, + env: { + PATH: `${fixture.bin}${path.delimiter}${process.env.PATH ?? ''}`, + HOME: fixture.home, + TMPDIR: fixture.root, + HTTP_PROXY: 'http://127.0.0.1:9', + HTTPS_PROXY: 'http://127.0.0.1:9', + NO_PROXY: '', + no_proxy: '', + NO_UPDATE_NOTIFIER: '1', + }, + maxBuffer: 2_000_000, + }, (error, stdout, stderr) => resolve({ + code: error && 'code' in error && typeof error.code === 'number' ? error.code : 0, + stdout, + stderr, + })); + child.stdin?.end(input); + }); +} + +async function latestState(repo: string): Promise { + const root = path.join(repo, '.orchestry', 'workflows'); + const ids = await fs.readdir(root); + expect(ids).toHaveLength(1); + return JSON.parse(await fs.readFile(path.join(root, ids[0]!, 'state.json'), 'utf8')); +} + +async function calls(home: string): Promise> { + try { + const content = await fs.readFile(path.join(home, 'native-role-calls.jsonl'), 'utf8'); + return content.trim().split('\n').filter(Boolean).map((line) => JSON.parse(line)); + } catch { + return []; + } +} + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await execFileAsync('git', args, { cwd }); + return stdout.trim(); +} + +async function exists(target: string): Promise { + return fs.access(target).then(() => true, () => false); +} diff --git a/test/unit/domain/native-role-workflow.test.ts b/test/unit/domain/native-role-workflow.test.ts new file mode 100644 index 0000000..2d5ea02 --- /dev/null +++ b/test/unit/domain/native-role-workflow.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest'; +import { validateWorkflowBindings } from '../../../src/domain/native-role-workflow.js'; + +const valid = { + supervisor: { cli: 'codex' as const, model: 'supervisor-model' }, + adviser: null, + implementer: { cli: 'claude' as const, model: 'implementer-model' }, + reviewer: { cli: 'codex' as const, model: 'reviewer-model' }, +}; + +describe('native role workflow bindings', () => { + it('accepts the supported direct role path', () => { + expect(() => validateWorkflowBindings(valid)).not.toThrow(); + }); + + it.each([ + ['supervisor', 'claude'], + ['implementer', 'codex'], + ['reviewer', 'claude'], + ['adviser', 'codex'], + ] as const)('rejects unsupported %s transport', (role, cli) => { + const bindings = { ...valid, [role]: { cli, model: 'model' } }; + expect(() => validateWorkflowBindings(bindings as typeof valid)).toThrow(); + }); + + it('rejects unsafe model arguments', () => { + expect(() => validateWorkflowBindings({ + ...valid, + supervisor: { cli: 'codex', model: '--dangerous flag' }, + })).toThrow('supervisor model is invalid'); + }); +});