From 9916d454990ea85f6f05ba56301ff24cee8c720d Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Wed, 29 Oct 2025 20:27:20 +0800 Subject: [PATCH] [feat] implement Pochi CLI for running evaluations with detailed logging and result handling --- lib/pochi-runner.ts | 582 ++++++++++++++++++++++++++++++++++++++++++++ pochi-cli.ts | 359 +++++++++++++++++++++++++++ 2 files changed, 941 insertions(+) create mode 100644 lib/pochi-runner.ts create mode 100755 pochi-cli.ts diff --git a/lib/pochi-runner.ts b/lib/pochi-runner.ts new file mode 100644 index 000000000..c13210381 --- /dev/null +++ b/lib/pochi-runner.ts @@ -0,0 +1,582 @@ +import fs from "fs/promises"; +import path from "path"; +import { spawn, ChildProcess } from "child_process"; +import { performance } from "perf_hooks"; +import { copyFolder, ensureSharedDependencies } from "./eval-runner"; + +export interface PochiResult { + success: boolean; + output: string; + error?: string; + duration: number; + buildSuccess?: boolean; + lintSuccess?: boolean; + testSuccess?: boolean; + buildOutput?: string; + lintOutput?: string; + testOutput?: string; + evalPath?: string; + timestamp?: string; +} + +export interface PochiEvalOptions { + timeout?: number; + verbose?: boolean; + debug?: boolean; + apiKey?: string; + model?: string; + outputFile?: string; + skipFileWrite?: boolean; +} + +export class PochiRunner { + private processes = new Map(); + private verbose: boolean; + private debug: boolean; + private apiKey?: string; + private model?: string; + + constructor(options: PochiEvalOptions = {}) { + this.verbose = options.verbose || false; + this.debug = options.debug || false; + this.apiKey = options.apiKey || process.env.POCHI_API_KEY; + this.model = options.model; + } + + async runPochiEval( + inputDir: string, + outputDir: string, + prompt: string, + timeout: number = 600000 // 10 minutes default + ): Promise { + const startTime = performance.now(); + + try { + // Ensure output directory exists and copy input files + await fs.mkdir(outputDir, { recursive: true }); + await copyFolder(inputDir, outputDir, true); // Exclude test files so pochi doesn't see them + + // Ensure shared dependencies are available + await ensureSharedDependencies(this.verbose); + + if (this.verbose) { + console.log(`๐Ÿค– Running Pochi on ${outputDir}...`); + console.log(`๐Ÿ“ Prompt: ${prompt}`); + console.log('โ”€'.repeat(80)); + } + + // Run Pochi with the prompt + const pochiResult = await this.executePochi(outputDir, prompt, timeout); + + if (!pochiResult.success) { + return { + success: false, + output: pochiResult.output, + error: pochiResult.error, + duration: performance.now() - startTime, + }; + } + + // Copy test files and eslint config back for evaluation + if (this.verbose) { + console.log('๐Ÿ“‹ Copying test files and eslint config back for evaluation...'); + } + await this.copyTestFilesBack(inputDir, outputDir); + + // Run evaluation (build, lint, test) on the modified code + const evalResults = await this.runEvaluation(outputDir); + + return { + success: true, + output: pochiResult.output, + duration: performance.now() - startTime, + buildSuccess: evalResults.buildSuccess, + lintSuccess: evalResults.lintSuccess, + testSuccess: evalResults.testSuccess, + buildOutput: evalResults.buildOutput, + lintOutput: evalResults.lintOutput, + testOutput: evalResults.testOutput, + }; + } catch (error) { + return { + success: false, + output: "", + error: error instanceof Error ? error.message : String(error), + duration: performance.now() - startTime, + }; + } finally { + // Clean up if not in debug mode + console.log(`๐Ÿงน Cleanup: debug=${this.debug}, outputDir=${outputDir}`); + if (!this.debug) { + console.log(`๐Ÿ—‘๏ธ Removing output directory...`); + try { + await fs.rm(outputDir, { recursive: true, force: true }); + console.log(`โœ… Output directory removed`); + } catch (error) { + console.log(`โš ๏ธ Cleanup error: ${error}`); + // Ignore cleanup errors + } + } else { + console.log(`๐Ÿ” Debug mode: preserving output directory`); + } + } + } + + private async executePochi( + projectDir: string, + prompt: string, + timeout: number + ): Promise<{ success: boolean; output: string; error?: string }> { + return new Promise((resolve, reject) => { + const processId = Math.random().toString(36).substr(2, 9); + const startTime = Date.now(); + + // Append instructions to prompt to prevent running dev servers + const enhancedPrompt = `${prompt} + +IMPORTANT: Do not run any pnpm, npm, or yarn commands (like pnpm dev, npm run dev, pnpm install, etc.). Do not start any development servers. Just make the necessary code changes to the files and exit when done. DO Not ask any followup questions either.`; + + // Prepare environment variables + const env = { ...process.env }; + if (this.apiKey) { + env.POCHI_API_KEY = this.apiKey; + } + + // Spawn Pochi process + // We'll pass the prompt via stdin to avoid escaping issues + const args: string[] = []; + + // Add model flag if specified + if (this.model) { + args.push('--model', this.model); + } + + console.log('๐Ÿš€ Spawning pochi process with:'); + console.log(' Command: pochi'); + console.log(' Args:', args); + console.log(' Working Directory:', projectDir); + console.log(' API Key present:', !!this.apiKey); + if (this.model) { + console.log(' Model:', this.model); + } + console.log(' Prompt length:', enhancedPrompt.length, 'chars'); + + const pochiProcess = spawn('pochi', args, { + cwd: projectDir, + env, + stdio: ['pipe', 'pipe', 'pipe'] // Use pipe for stdin to send the prompt + }); + this.processes.set(processId, pochiProcess); + + // Send the enhanced prompt via stdin and close it + if (pochiProcess.stdin) { + pochiProcess.stdin.write(enhancedPrompt); + pochiProcess.stdin.end(); + } + + let stdout = ''; + let stderr = ''; + let lastOutputTime = startTime; + let resolved = false; + + const idleTimeoutMs = 90000; // 90 second idle timeout + let idleTimeoutHandle: NodeJS.Timeout | null = null; + + function resolveOnce(result: { success: boolean; output: string; error?: string }) { + if (resolved) return; + resolved = true; + clearTimeout(absoluteTimeoutId); + if (idleTimeoutHandle) clearTimeout(idleTimeoutHandle); + clearInterval(heartbeat); + resolve(result); + } + + function resetIdleTimeout() { + if (idleTimeoutHandle) clearTimeout(idleTimeoutHandle); + + idleTimeoutHandle = setTimeout(() => { + const sinceLastOutput = Date.now() - lastOutputTime; + console.log(`โฑ๏ธ Idle timeout reached (${(sinceLastOutput / 1000).toFixed(1)}s since last output)`); + console.log(`๐Ÿ›‘ Forcefully terminating pochi process ${pochiProcess.pid}...`); + pochiProcess.kill('SIGTERM'); + + setTimeout(() => { + if (!resolved) { + console.log(`๐Ÿ›‘ Process didn't respond to SIGTERM, using SIGKILL...`); + pochiProcess.kill('SIGKILL'); + } + }, 5000); + }, idleTimeoutMs); + } + + // Start idle timeout + resetIdleTimeout(); + + // Set up a heartbeat to show the process is still running + const heartbeat = setInterval(() => { + const elapsed = Date.now() - startTime; + const sinceLastOutput = Date.now() - lastOutputTime; + console.log(`โณ Pochi still running... (${(elapsed / 1000).toFixed(1)}s elapsed, ${(sinceLastOutput / 1000).toFixed(1)}s since last output)`); + }, 5000); // Log every 5 seconds + + pochiProcess.stdout?.on('data', (data) => { + const output = data.toString(); + lastOutputTime = Date.now(); + resetIdleTimeout(); + // Always log stdout in real-time to help debug + process.stdout.write(`[pochi stdout] ${output}`); + // Also log raw bytes to see if there are hidden characters + if (this.verbose) { + console.log(`[DEBUG] stdout bytes: ${JSON.stringify(output)}`); + } + stdout += output; + }); + + pochiProcess.stderr?.on('data', (data) => { + const output = data.toString(); + lastOutputTime = Date.now(); + resetIdleTimeout(); + // Always log stderr in real-time to help debug + process.stderr.write(`[pochi stderr] ${output}`); + if (this.verbose) { + console.log(`[DEBUG] stderr bytes: ${JSON.stringify(output)}`); + } + stderr += output; + }); + + const absoluteTimeoutId = setTimeout(() => { + console.log(`โฑ๏ธ Absolute timeout reached (${timeout}ms)`); + pochiProcess.kill('SIGTERM'); + setTimeout(() => { + pochiProcess.kill('SIGKILL'); + }, 5000); + resolveOnce({ + success: false, + output: stdout, + error: `Pochi process timed out after ${timeout}ms` + }); + }, timeout); + + pochiProcess.on('exit', (code, signal) => { + const elapsed = Date.now() - startTime; + console.log(`โœ“ Pochi process exited with code: ${code}, signal: ${signal} after ${(elapsed / 1000).toFixed(1)}s`); + + resolveOnce({ + success: code === 0 && !signal, + output: stdout, + error: signal + ? `Pochi process killed by signal ${signal}` + : code !== 0 + ? stderr || `Pochi process exited with code ${code}` + : undefined + }); + }); + + pochiProcess.on('error', (error) => { + console.log(`โŒ Pochi process error: ${error.message}`); + resolveOnce({ + success: false, + output: stdout, + error: error.message + }); + }); + + console.log(`๐Ÿ“ Pochi process spawned with PID: ${pochiProcess.pid}`); + }); + } + + private async copyTestFilesBack(inputDir: string, outputDir: string): Promise { + const entries = await fs.readdir(inputDir, { withFileTypes: true }); + + for (const entry of entries) { + if (entry.name === "node_modules") { + continue; + } + + const isTestFile = entry.name.endsWith(".test.tsx") || + entry.name.endsWith(".test.ts") || + entry.name.endsWith(".spec.tsx") || + entry.name.endsWith(".spec.ts") || + entry.name.endsWith(".test.jsx") || + entry.name.endsWith(".test.js") || + entry.name.endsWith(".spec.jsx") || + entry.name.endsWith(".spec.js"); + const isTestDir = entry.name === "__tests__" || + entry.name === "test" || + entry.name === "tests"; + const isEslintConfig = entry.name === ".eslintrc.json" || + entry.name === ".eslintrc.js" || + entry.name === ".eslintrc.cjs" || + entry.name === ".eslintrc.yml" || + entry.name === ".eslintrc.yaml" || + entry.name === "eslint.config.js" || + entry.name === "eslint.config.mjs" || + entry.name === "eslint.config.cjs"; + + const srcPath = path.join(inputDir, entry.name); + const destPath = path.join(outputDir, entry.name); + + try { + if (isTestFile || isEslintConfig) { + // Copy the test file or eslint config + await fs.copyFile(srcPath, destPath); + } else if (entry.isDirectory() && isTestDir) { + // Copy the test directory + await copyFolder(srcPath, destPath, false); // Don't exclude anything when copying test dirs + } else if (entry.isDirectory()) { + // Recursively copy test files from subdirectories + await this.copyTestFilesBack(srcPath, destPath); + } + } catch (error) { + // Ignore errors (e.g., directory doesn't exist in output) + } + } + } + + private async runEvaluation(projectDir: string): Promise<{ + buildSuccess: boolean; + lintSuccess: boolean; + testSuccess: boolean; + buildOutput: string; + lintOutput: string; + testOutput: string; + }> { + let buildSuccess = false; + let buildOutput = ""; + let lintSuccess = false; + let lintOutput = ""; + let testSuccess = false; + let testOutput = ""; + + // Run next build + try { + if (this.verbose) { + console.log("Running build..."); + } + buildOutput = await this.execCommand( + `cd "${projectDir}" && ../../node_modules/.bin/next build`, + 60000 + ); + buildSuccess = true; + if (this.verbose) { + console.log("โœ… Build completed"); + } + } catch (error) { + if (error && typeof error === "object" && "stdout" in error) { + buildOutput += (error as any).stdout || ""; + if ((error as any).stderr) { + buildOutput += "\n" + (error as any).stderr; + } + } else { + buildOutput += error instanceof Error ? error.message : String(error); + } + if (this.verbose) { + console.log("โŒ Build failed"); + } + } + + // Run linting + try { + if (this.verbose) { + console.log("Running lint..."); + } + + // Check if .eslintrc.json exists, create a basic one if not + const eslintConfigPath = path.join(projectDir, ".eslintrc.json"); + const eslintConfigExists = await fs + .stat(eslintConfigPath) + .then(() => true) + .catch(() => false); + + if (!eslintConfigExists) { + const basicEslintConfig = { + extends: "next/core-web-vitals", + }; + await fs.writeFile( + eslintConfigPath, + JSON.stringify(basicEslintConfig, null, 2), + ); + } + + lintOutput = await this.execCommand( + `cd "${projectDir}" && ../../node_modules/.bin/next lint`, + 30000 + ); + lintSuccess = true; + if (this.verbose) { + console.log("โœ… Lint completed"); + } + } catch (error) { + if (error && typeof error === "object" && "stdout" in error) { + lintOutput = (error as any).stdout || ""; + if ((error as any).stderr) { + lintOutput += "\n" + (error as any).stderr; + } + } else { + lintOutput = error instanceof Error ? error.message : String(error); + } + if (this.verbose) { + console.log("โŒ Lint failed"); + } + } + + // Run tests + try { + if (this.verbose) { + console.log("Running tests..."); + } + testOutput = await this.execCommand( + `cd "${projectDir}" && ../../node_modules/.bin/vitest run`, + 30000 + ); + testSuccess = true; + if (this.verbose) { + console.log("โœ… Tests completed"); + } + } catch (error) { + if (error && typeof error === "object" && "stdout" in error) { + testOutput = (error as any).stdout || ""; + if ((error as any).stderr) { + testOutput += "\n" + (error as any).stderr; + } + } else { + testOutput = error instanceof Error ? error.message : String(error); + } + if (this.verbose) { + console.log("โŒ Tests failed"); + } + } + + return { + buildSuccess, + buildOutput, + lintSuccess, + lintOutput, + testSuccess, + testOutput, + }; + } + + private async execCommand(command: string, timeout: number): Promise { + return new Promise((resolve, reject) => { + const { exec } = require('child_process'); + const process = exec(command, { + maxBuffer: 10 * 1024 * 1024, // 10MB buffer + timeout + }, (error: any, stdout: string, stderr: string) => { + if (error) { + error.stdout = stdout; + error.stderr = stderr; + reject(error); + } else { + resolve(stdout); + } + }); + }); + } + + async cleanup(): Promise { + const promises = Array.from(this.processes.entries()).map( + ([processId, process]) => + new Promise((resolve) => { + process.kill('SIGTERM'); + process.on('exit', () => { + this.processes.delete(processId); + resolve(); + }); + // Force kill after 5 seconds if not terminated + setTimeout(() => { + process.kill('SIGKILL'); + this.processes.delete(processId); + resolve(); + }, 5000); + }) + ); + await Promise.all(promises); + } +} + +export async function runPochiEval( + evalPath: string, + options: PochiEvalOptions = {} +): Promise { + const evalsDir = path.join(process.cwd(), "evals"); + const fullEvalPath = path.join(evalsDir, evalPath); + + // Check if the eval directory exists + const evalStat = await fs.stat(fullEvalPath).catch(() => null); + if (!evalStat || !evalStat.isDirectory()) { + throw new Error(`Eval directory not found: ${evalPath}`); + } + + // Look for input directory + const inputDir = path.join(fullEvalPath, "input"); + const inputExists = await fs + .stat(inputDir) + .then((s) => s.isDirectory()) + .catch(() => false); + if (!inputExists) { + throw new Error(`No input directory found in ${evalPath}`); + } + + // Read prompt from prompt.md + const promptFile = path.join(fullEvalPath, "prompt.md"); + const promptExists = await fs + .stat(promptFile) + .then((s) => s.isFile()) + .catch(() => false); + if (!promptExists) { + throw new Error(`No prompt.md file found in ${evalPath}`); + } + + const prompt = await fs.readFile(promptFile, "utf8"); + const outputDir = path.join(fullEvalPath, "output-pochi"); + + const runner = new PochiRunner(options); + + try { + const result = await runner.runPochiEval(inputDir, outputDir, prompt, options.timeout); + + // Add evalPath and timestamp to result + const timestamp = new Date().toISOString(); + const enrichedResult: PochiResult = { + ...result, + evalPath, + timestamp, + }; + + // Write results to file unless skipFileWrite is true + if (!options.skipFileWrite) { + // Determine output file path + let outputFile = options.outputFile; + if (!outputFile) { + // Create default output file in results directory + const resultsDir = path.join(process.cwd(), "results"); + await fs.mkdir(resultsDir, { recursive: true }); + const sanitizedEvalPath = evalPath.replace(/\//g, "-"); + const timestampStr = Date.now(); + outputFile = path.join(resultsDir, `pochi-${sanitizedEvalPath}-${timestampStr}.json`); + } + + // Write results to file + try { + await fs.writeFile( + outputFile, + JSON.stringify(enrichedResult, null, 2), + "utf-8" + ); + console.log(`๐Ÿ“ Results written to: ${outputFile}`); + } catch (error) { + console.error( + `โš ๏ธ Failed to write results to file: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + return enrichedResult; + } finally { + await runner.cleanup(); + } +} diff --git a/pochi-cli.ts b/pochi-cli.ts new file mode 100755 index 000000000..7a36904c3 --- /dev/null +++ b/pochi-cli.ts @@ -0,0 +1,359 @@ +#!/usr/bin/env bun + +import fs from "fs/promises"; +import path from "path"; +import { parseArgs } from "util"; +import { runPochiEval, PochiResult } from "./lib/pochi-runner"; + +const { values, positionals } = parseArgs({ + args: process.argv.slice(2), + options: { + help: { type: "boolean", short: "h" }, + eval: { type: "string", short: "e" }, + all: { type: "boolean", short: "a" }, + verbose: { type: "boolean", short: "v" }, + debug: { type: "boolean" }, + timeout: { type: "string", short: "t" }, + "api-key": { type: "string" }, + model: { type: "string", short: "m" }, + "output-file": { type: "string" }, + }, + allowPositionals: true, +}); + +function showHelp() { + console.log(` +Pochi Evals CLI + +Usage: + pochi-cli.ts [options] [eval-path] + +Options: + -h, --help Show this help message + -e, --eval Run a specific eval by path + -a, --all Run all evals with Pochi + -v, --verbose Show detailed logs during eval execution + --debug Persist output folders for debugging (don't clean up) + -t, --timeout Timeout in milliseconds (default: 600000 = 10 minutes) + --api-key Pochi API key (or use POCHI_API_KEY env var) + -m, --model Model to use (e.g., gpt-4o, claude-sonnet-4-20250514) + --output-file Custom path for results file (default: results/pochi-*.json) + +Results are automatically written to the results/ directory. + +Examples: + # Run a specific eval (results auto-saved to results/pochi-*.json) + bun pochi-cli.ts --eval 001-server-component + + # Run eval by positional argument + bun pochi-cli.ts 001-server-component + + # Run with verbose output and custom timeout + bun pochi-cli.ts --eval 001-server-component --verbose --timeout 600000 + + # Run with specific model + bun pochi-cli.ts --eval 001-server-component --model gpt-4o + + # Run all evals (results auto-saved to results/pochi-all-*.json) + bun pochi-cli.ts --all + + # Debug mode - keep output folders for inspection + bun pochi-cli.ts --eval 001-server-component --debug + + # Write results to custom location + bun pochi-cli.ts --eval 001-server-component --output-file my-results.json +`); +} + +async function getAllEvals(): Promise { + const evalsDir = path.join(process.cwd(), "evals"); + const entries = await fs.readdir(evalsDir, { withFileTypes: true }); + + const evals: string[] = []; + + for (const entry of entries) { + if (entry.isDirectory() && /^\d+/.test(entry.name)) { + const evalPath = path.join(evalsDir, entry.name); + // Check if it has both input/ directory and prompt.md + const hasInput = await fs + .stat(path.join(evalPath, "input")) + .then((s) => s.isDirectory()) + .catch(() => false); + const hasPrompt = await fs + .stat(path.join(evalPath, "prompt.md")) + .then((s) => s.isFile()) + .catch(() => false); + + if (hasInput && hasPrompt) { + evals.push(entry.name); + } + } + } + + return evals.sort(); +} + +function formatDuration(ms: number): string { + if (ms < 1000) { + return `${Math.round(ms)}ms`; + } else { + const seconds = ms / 1000; + return `${seconds.toFixed(1)}s`; + } +} + +function displayResult(evalPath: string, result: PochiResult) { + console.log("\n๐Ÿ“Š Pochi Results:"); + console.log("โ•".repeat(80)); + + const evalColWidth = Math.max(25, evalPath.length); + const header = `| ${"Eval".padEnd(evalColWidth)} | Result | Build | Lint | Tests | Duration |`; + const separator = `|${"-".repeat(evalColWidth + 2)}|------------|-------|-------|-------|----------|`; + + console.log(header); + console.log(separator); + + const name = evalPath.padEnd(evalColWidth); + const build = result.buildSuccess ? "โœ…" : "โŒ"; + const lint = result.lintSuccess ? "โœ…" : "โŒ"; + const tests = result.testSuccess ? "โœ…" : "โŒ"; + const allPassed = result.buildSuccess && result.lintSuccess && result.testSuccess; + const resultStatus = allPassed ? "โœ… PASS" : "โŒ FAIL"; + const duration = formatDuration(result.duration); + + console.log( + `| ${name} | ${resultStatus.padEnd(10)} | ${build} | ${lint} | ${tests} | ${duration.padEnd(8)} |` + ); + + console.log("โ•".repeat(80)); + + if (!allPassed || !result.success) { + console.log("\nโŒ Error Details:"); + console.log("โ”€".repeat(80)); + + if (result.error) { + console.log(`Pochi Error: ${result.error}`); + } + + if (!result.buildSuccess && result.buildOutput) { + console.log(`Build Error:\n${result.buildOutput.slice(-1000)}`); + } + + if (!result.lintSuccess && result.lintOutput) { + console.log(`Lint Error:\n${result.lintOutput.slice(-1000)}`); + } + + if (!result.testSuccess && result.testOutput) { + console.log(`Test Error:\n${result.testOutput.slice(-1000)}`); + } + } + + console.log("โ•".repeat(80)); +} + +function displayResultsTable(results: { evalPath: string; result: PochiResult }[]) { + const totalTests = results.length; + console.log(`\n๐Ÿ“Š Pochi Results Summary (${totalTests} Tests):`); + console.log("โ•".repeat(120)); + + const header = `| ${"Eval".padEnd(25)} | Result | Build | Lint | Tests | Duration |`; + const separator = `|${"-".repeat(27)}|------------|-------|-------|-------|----------|`; + + console.log(header); + console.log(separator); + + const failedEvals: Array<{ + evalPath: string; + buildError?: string; + lintError?: string; + testError?: string; + pochiError?: string; + }> = []; + + let passedEvals = 0; + + for (const { evalPath, result } of results) { + const name = evalPath.padEnd(25); + const build = result.buildSuccess ? "โœ…" : "โŒ"; + const lint = result.lintSuccess ? "โœ…" : "โŒ"; + const tests = result.testSuccess ? "โœ…" : "โŒ"; + const allPassed = result.success && result.buildSuccess && result.lintSuccess && result.testSuccess; + const resultStatus = allPassed ? "โœ… PASS" : "โŒ FAIL"; + const duration = formatDuration(result.duration); + + if (allPassed) { + passedEvals++; + } + + console.log( + `| ${name} | ${resultStatus.padEnd(10)} | ${build} | ${lint} | ${tests} | ${duration.padEnd(8)} |` + ); + + // Collect errors for failed evals + if (!allPassed) { + const errors: any = { evalPath }; + + if (result.error) { + errors.pochiError = result.error; + } + + if (!result.buildSuccess && result.buildOutput) { + errors.buildError = result.buildOutput.slice(-500); + } + + if (!result.lintSuccess && result.lintOutput) { + errors.lintError = result.lintOutput.slice(-500); + } + + if (!result.testSuccess && result.testOutput) { + errors.testError = result.testOutput.slice(-500); + } + + failedEvals.push(errors); + } + } + + console.log("โ•".repeat(120)); + + // Summary stats + console.log(`\n๐Ÿ“ˆ Summary: ${passedEvals}/${totalTests} evals passed`); + + // Display error summaries + if (failedEvals.length > 0) { + console.log("\nโŒ Error Summaries:"); + console.log("โ”€".repeat(120)); + + for (const failed of failedEvals) { + console.log(`\n${failed.evalPath}:`); + + if (failed.pochiError) { + console.log(` Pochi: ${failed.pochiError}`); + } + + if (failed.buildError) { + console.log(` Build: ${failed.buildError}`); + } + + if (failed.lintError) { + console.log(` Lint: ${failed.lintError}`); + } + + if (failed.testError) { + console.log(` Tests: ${failed.testError}`); + } + } + } +} + +async function main() { + if (values.help) { + showHelp(); + return; + } + + // Check for API key (optional for Pochi) + const apiKey = values["api-key"] || process.env.POCHI_API_KEY; + + const evalOptions = { + verbose: values.verbose || false, + debug: values.debug || false, + timeout: values.timeout ? parseInt(values.timeout) : 600000, // 10 minutes default + apiKey, + model: values.model, + outputFile: values["output-file"], + }; + + if (values.all) { + const allEvals = await getAllEvals(); + console.log(`Running ${allEvals.length} evals with Pochi...${values.model ? ` (model: ${values.model})` : ''}\n`); + + const results: { evalPath: string; result: PochiResult }[] = []; + + // Don't write individual files - we'll write all results to one file at the end + const individualEvalOptions = { ...evalOptions, skipFileWrite: true }; + + for (const evalPath of allEvals) { + try { + console.log(`๐Ÿš€ Running ${evalPath}...`); + const result = await runPochiEval(evalPath, individualEvalOptions); + results.push({ evalPath, result }); + + const status = result.success && result.buildSuccess && result.lintSuccess && result.testSuccess + ? "โœ… PASS" + : "โŒ FAIL"; + console.log(`${status} ${evalPath} (${formatDuration(result.duration)})`); + + } catch (error) { + const errorResult: PochiResult = { + success: false, + output: "", + error: error instanceof Error ? error.message : String(error), + duration: 0, + }; + results.push({ evalPath, result: errorResult }); + console.log(`โŒ FAIL ${evalPath} - ${errorResult.error}`); + } + } + + displayResultsTable(results); + + // Determine output file for all results + let allResultsFile = evalOptions.outputFile; + if (!allResultsFile) { + // Create default output file in results directory + const resultsDir = path.join(process.cwd(), "results"); + await fs.mkdir(resultsDir, { recursive: true }); + const timestamp = Date.now(); + allResultsFile = path.join(resultsDir, `pochi-all-${timestamp}.json`); + } + + // Write all results to file + try { + await fs.writeFile( + allResultsFile, + JSON.stringify(results, null, 2), + "utf-8" + ); + console.log(`\n๐Ÿ“ All results written to: ${allResultsFile}`); + } catch (error) { + console.error( + `โš ๏ธ Failed to write results to file: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + + return; + } + + const evalPath = values.eval || positionals[0]; + if (!evalPath) { + console.error("โŒ Error: No eval specified. Use --eval , provide a positional argument, or use --all"); + console.log("\nAvailable evals:"); + const allEvals = await getAllEvals(); + allEvals.forEach((evalName) => console.log(` ${evalName}`)); + process.exit(1); + } + + console.log(`๐Ÿš€ Running Pochi eval: ${evalPath}${values.model ? ` (model: ${values.model})` : ''}`); + + try { + const result = await runPochiEval(evalPath, evalOptions); + displayResult(evalPath, result); + + const success = result.success && result.buildSuccess && result.lintSuccess && result.testSuccess; + process.exit(success ? 0 : 1); + + } catch (error) { + console.error(`โŒ Error: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } +} + +// @ts-ignore +if (import.meta.main) { + main().catch((error) => { + console.error("Unexpected error:", error); + process.exit(1); + }); +}