diff --git a/esure-cli/README.md b/esure-cli/README.md index 68f1bd5..4bb80b5 100644 --- a/esure-cli/README.md +++ b/esure-cli/README.md @@ -267,13 +267,18 @@ Executes a scenario on Stellar Testnet. - ``: Bundled scenario ID (e.g., `xlm-payment`) or path to scenario file **Options:** - -- `--output json`: Output execution report in JSON format +- `--output, -o json`: Output execution report in JSON format +- `--timeout-ms `: Overall run timeout in milliseconds (default: 120000) +- `--ascii`: Use ASCII status markers (PASS/FAIL) instead of Unicode glyphs **Exit codes:** - `0`: All steps and assertions passed -- `1`: One or more steps or assertions failed, or execution error +- `1`: One or more steps or assertions failed, execution error, or run timeout +- `130`: Interrupted by SIGINT +- `143`: Terminated by SIGTERM + +When `--output json` is requested, argument errors and cancellation responses are emitted to `stderr` in a structured JSON error envelope (`{"error": {"code": "...", "message": "..."}}`). ### `esure run-suite [...]` diff --git a/esure-cli/src/cli.ts b/esure-cli/src/cli.ts index fb9a67e..3c84b51 100644 --- a/esure-cli/src/cli.ts +++ b/esure-cli/src/cli.ts @@ -1,19 +1,57 @@ #!/usr/bin/env node -import { readFile, stat } from "node:fs/promises"; -import { extname, resolve } from "node:path"; +import { existsSync, readFileSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; import { SCENARIO_LIMITS, ScenarioValidationError } from "@esure/backend/src/scenario-schema.js"; import { parseScenario } from "@esure/backend/src/scenario-loader.js"; import { createScenarioRegistry } from "@esure/backend/src/scenarios.js"; import { StellarTestnetGateway } from "@esure/backend/src/stellar-gateway.js"; +import { SafeRunError } from "@esure/backend/src/errors.js"; +import { parse as parseYaml } from "yaml"; import type { ValidatedScenario } from "@esure/backend/src/domain.js"; import type { LedgerExecution, LedgerExecutionOptions } from "@esure/backend/src/domain.js"; -const VERSION = "0.1.0"; +// ───────────────────────────────────────────────────────────────────────────── +// Version resolution (#57) +// ───────────────────────────────────────────────────────────────────────────── + +export function resolveVersion(): string { + try { + let dir = dirname(fileURLToPath(import.meta.url)); + for (let i = 0; i < 4; i++) { + const candidate = join(dir, "package.json"); + if (existsSync(candidate)) { + try { + const pkg = JSON.parse(readFileSync(candidate, "utf8")); + if (pkg.name === "@esure/cli" && typeof pkg.version === "string") { + return pkg.version; + } + } catch { + // ignore JSON parse error and keep walking up + } + } + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + } catch { + // fallback if import.meta.url is unavailable + } + return "0.1.0"; +} + +const VERSION = resolveVersion(); const BUNDLED_SCENARIO_DIR_TIMEOUT_MS = 30_000; const MAX_REPORT_FILE_BYTES = 5 * 1024 * 1024; +/** Default overall run timeout in milliseconds (#55). */ +export const DEFAULT_CLI_TIMEOUT_MS = 120_000; +export const MIN_CLI_TIMEOUT_MS = 1; +export const MAX_CLI_TIMEOUT_MS = 600_000; + // ───────────────────────────────────────────────────────────────────────────── // Shared types // ───────────────────────────────────────────────────────────────────────────── @@ -24,8 +62,8 @@ interface CliOptions { version?: boolean; /** #68 — use ASCII status markers (PASS/FAIL) instead of Unicode glyphs */ ascii?: boolean; - /** #53 — explicit local file path for run command */ - file?: string; + /** #55 — overall run timeout in milliseconds */ + timeoutMs?: number; } // ───────────────────────────────────────────────────────────────────────────── @@ -45,6 +83,25 @@ function statusMarker(status: "passed" | "failed" | string, m: Markers): string return status === "passed" ? m.pass : m.fail; } +// ───────────────────────────────────────────────────────────────────────────── +// Output and error helpers (#54, #55, #56) +// ───────────────────────────────────────────────────────────────────────────── + +function reportArgumentError( + message: string, + usageHint: string | undefined, + output: "human" | "json" | undefined, +): void { + if (output === "json") { + console.error(JSON.stringify({ error: { code: "INVALID_ARGUMENT", message } })); + } else { + console.error(`Error: ${message}`); + if (usageHint) { + console.error(usageHint); + } + } +} + // ───────────────────────────────────────────────────────────────────────────── // Help & version // ───────────────────────────────────────────────────────────────────────────── @@ -66,12 +123,12 @@ Usage: esure doctor Diagnose the local installation Options: - --file (run) Explicitly specify a local scenario file path - --output Output format: human (default) or json - --ascii Use ASCII status markers (PASS/FAIL) instead of Unicode glyphs - --fail-fast (run-suite) Stop after first failure - --help Show this help message - --version Show version information + --output, -o Output format: human (default) or json + --timeout-ms Overall run timeout in milliseconds (default: 120000) + --ascii Use ASCII status markers (PASS/FAIL) instead of Unicode glyphs + --fail-fast (run-suite) Stop after first failure + --help, -h Show this help message + --version, -v Show version information Examples: esure list @@ -80,7 +137,7 @@ Examples: esure report validate ./report.json --output json esure run xlm-payment esure run ./my-scenario.yaml --output json - esure run --file ./my-scenario.yaml + esure run xlm-payment --timeout-ms 30000 esure run-suite xlm-payment path-payment --fail-fast Documentation: https://github.com/Esureorg/Esure @@ -100,68 +157,85 @@ function parseCliArgs(): { args: string[]; options: CliOptions; failFast: boolean; - parseError?: string; + parseError?: { message: string; usage?: string }; } { - let parsedValues: Record; - let parsedPositionals: string[]; + // Pre-scan for requested output mode so parse errors can be formatted as JSON (#54) + const argv = process.argv.slice(2); + let requestedOutput: "human" | "json" = "human"; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--output=json" || arg === "-o=json") { + requestedOutput = "json"; + } else if ((arg === "--output" || arg === "-o") && argv[i + 1] === "json") { + requestedOutput = "json"; + } + } + let parsed: ReturnType; try { - const parsed = parseArgs({ + parsed = parseArgs({ options: { - output: { type: "string" }, + output: { type: "string", short: "o" }, help: { type: "boolean", short: "h" }, version: { type: "boolean", short: "v" }, ascii: { type: "boolean" }, "fail-fast": { type: "boolean" }, - "report-file": { type: "string" }, - junit: { type: "string" }, - "junit-file": { type: "string" }, - csv: { type: "string" }, - overwrite: { type: "boolean" }, - force: { type: "boolean" }, + "timeout-ms": { type: "string" }, }, allowPositionals: true, strict: true, }); - parsedValues = parsed.values; - parsedPositionals = parsed.positionals; - } catch (err: any) { + } catch (error) { return { - command: undefined, args: [], - options: { output: "human", help: false, version: false, ascii: false }, + options: { output: requestedOutput }, failFast: false, - parseError: err.message || "Invalid arguments", + parseError: { + message: error instanceof Error ? error.message : String(error), + usage: 'Run "esure --help" for usage information', + }, }; } - if (parsedValues.output !== undefined && parsedValues.output !== "human" && parsedValues.output !== "json") { - return { - command: parsedPositionals[0], - args: parsedPositionals.slice(1), - options: { output: "human", help: false, version: false, ascii: false }, - failFast: false, - parseError: `Invalid output format "${parsedValues.output}". Supported formats: human, json`, - }; + const { values, positionals } = parsed; + + let output: "human" | "json" = "human"; + if (values.output !== undefined) { + if (values.output === "json") { + output = "json"; + } else if (values.output === "human") { + output = "human"; + } else { + return { + args: [], + options: { output: requestedOutput }, + failFast: false, + parseError: { + message: `Invalid output format "${values.output}". Supported formats are "human" and "json".`, + }, + }; + } + } + + let timeoutMs: number | undefined; + if (values["timeout-ms"] !== undefined) { + const parsedTimeout = Number(values["timeout-ms"]); + if ( + !Number.isInteger(parsedTimeout) || + parsedTimeout < MIN_CLI_TIMEOUT_MS || + parsedTimeout > MAX_CLI_TIMEOUT_MS + ) { + return { + args: [], + options: { output }, + failFast: false, + parseError: { + message: `--timeout-ms must be an integer between ${MIN_CLI_TIMEOUT_MS} and ${MAX_CLI_TIMEOUT_MS}`, + }, + }; + } + timeoutMs = parsedTimeout; } - const { values, positionals } = parseArgs({ - options: { - output: { type: "string" }, - help: { type: "boolean", short: "h" }, - version: { type: "boolean", short: "v" }, - ascii: { type: "boolean" }, - file: { type: "string" }, - "fail-fast": { type: "boolean" }, - "report-file": { type: "string" }, - junit: { type: "string" }, - "junit-file": { type: "string" }, - csv: { type: "string" }, - overwrite: { type: "boolean" }, - force: { type: "boolean" }, - }, - allowPositionals: true, - strict: false, - }); const rawJunit = typeof values.junit === "string" @@ -174,11 +248,11 @@ function parseCliArgs(): { command: positionals[0], args: positionals.slice(1), options: { - output: (values.output as "human" | "json") || "human", + output, help: Boolean(values.help), version: Boolean(values.version), ascii: Boolean(values.ascii), - file: typeof values.file === "string" ? values.file : undefined, + timeoutMs, }, failFast: Boolean(values["fail-fast"]), }; @@ -223,15 +297,23 @@ async function loadScenarioFile(filePath: string): Promise { // ───────────────────────────────────────────────────────────────────────────── /** Build a LedgerExecutionOptions object that prints progress in human mode (#66). */ -function buildExecutionOptions(options: CliOptions): LedgerExecutionOptions { - const execOptions: LedgerExecutionOptions = { stepTimeoutMs: BUNDLED_SCENARIO_DIR_TIMEOUT_MS }; +function buildExecutionOptions( + options: CliOptions, + signal?: AbortSignal, +): LedgerExecutionOptions { + const execOptions: LedgerExecutionOptions = { + stepTimeoutMs: BUNDLED_SCENARIO_DIR_TIMEOUT_MS, + signal, + }; if (options.output === "human") { const m = markers(options.ascii ?? false); execOptions.onStep = (step) => { console.log(` ${statusMarker(step.status, m)} step ${step.id}: ${step.message}`); }; execOptions.onAssertion = (assertion) => { - console.log(` ${statusMarker(assertion.status, m)} check ${assertion.type}: ${assertion.message}`); + console.log( + ` ${statusMarker(assertion.status, m)} check ${assertion.type}: ${assertion.message}`, + ); }; } return execOptions; @@ -274,249 +356,132 @@ function buildRunReport( } // ───────────────────────────────────────────────────────────────────────────── -// Artifact writing & JUnit export (#59, #60, #62) +// Signal & timeout execution runner (#55, #56) // ───────────────────────────────────────────────────────────────────────────── -function escapeCsv(value: unknown): string { - const str = String(value ?? ""); - if (str.includes(",") || str.includes('"') || str.includes("\n")) { - return `"${str.replace(/"/g, '""')}"`; - } - return str; +interface ExecutionRunnerResult { + result?: T; + exitCode?: number; } -export function generateCsv(runs: ScenarioRunResultForJunit[]): string { - const rows: string[] = [ - ["Scenario", "Type", "Status", "Message"].map(escapeCsv).join(","), - ]; - - for (const run of runs) { - if (run.execution) { - for (const assertion of run.execution.assertions) { - rows.push( - [run.scenarioId, assertion.type, assertion.status, assertion.message] - .map(escapeCsv) - .join(","), - ); - } - } - } - - return rows.join("\n") + "\n"; -} - -export function escapeXml(unsafe: string): string { - return String(unsafe) - .replace(/&/g, "&") - .replace(//g, ">") - .replace(/"/g, """) - .replace(/'/g, "'"); -} - -export interface ScenarioRunResultForJunit { - scenarioId: string; - execution?: LedgerExecution; - error?: { message: string }; -} - -export function generateJunitXml(runs: ScenarioRunResultForJunit[]): string { - let totalTests = 0; - let totalFailures = 0; - let totalErrors = 0; - - const testsuiteXmls: string[] = []; - - for (const run of runs) { - let suiteTests = 0; - let suiteFailures = 0; - let suiteErrors = 0; - const testcaseXmls: string[] = []; - - if (run.execution) { - for (const step of run.execution.steps) { - suiteTests += 1; - const name = `step ${step.id}: ${step.message || step.type}`; - if (step.status === "passed") { - testcaseXmls.push( - ` `, - ); - } else { - suiteFailures += 1; - testcaseXmls.push( - ` \n ${escapeXml(step.message)}\n `, - ); - } - } - - for (const assertion of run.execution.assertions) { - suiteTests += 1; - const name = `assertion ${assertion.type}: ${assertion.message}`; - if (assertion.status === "passed") { - testcaseXmls.push( - ` `, - ); - } else { - suiteFailures += 1; - testcaseXmls.push( - ` \n ${escapeXml(assertion.message)}\n `, - ); - } - } +async function runWithSignalsAndTimeout( + timeoutMs: number, + options: CliOptions, + action: (signal: AbortSignal) => Promise, +): Promise> { + const controller = new AbortController(); + let timedOut = false; + let receivedSignal: "SIGINT" | "SIGTERM" | undefined; + let timer: NodeJS.Timeout | undefined; + + const onSigint = () => { + if (receivedSignal === "SIGINT") { + process.exit(130); } + receivedSignal = "SIGINT"; + controller.abort(); + }; - if (run.error) { - suiteTests += 1; - suiteErrors += 1; - testcaseXmls.push( - ` \n ${escapeXml(run.error.message)}\n `, - ); + const onSigterm = () => { + if (receivedSignal === "SIGTERM") { + process.exit(143); } + receivedSignal = "SIGTERM"; + controller.abort(); + }; - totalTests += suiteTests; - totalFailures += suiteFailures; - totalErrors += suiteErrors; - - const testcases = testcaseXmls.length > 0 ? `\n${testcaseXmls.join("\n")}\n ` : ""; - testsuiteXmls.push( - ` ${testcases}`, - ); - } - - const suites = testsuiteXmls.length > 0 ? `\n${testsuiteXmls.join("\n")}\n` : "\n"; - return `\n${suites}\n`; -} - -export async function writeArtifactFile( - filePath: string, - content: string, - overwrite: boolean, - artifactType: "report" | "JUnit" | "CSV" | "scaffold", -): Promise { - const absolutePath = resolve(filePath); - - let exists = false; - try { - await access(absolutePath); - exists = true; - } catch { - exists = false; - } - - if (exists && !overwrite) { - const typeLabel = - artifactType === "report" ? "Report" : artifactType === "JUnit" ? "JUnit" : artifactType === "CSV" ? "CSV" : "Scenario"; - throw new Error(`${typeLabel} file "${filePath}" already exists. Use --overwrite to replace it.`); - } + process.on("SIGINT", onSigint); + process.on("SIGTERM", onSigterm); - const dir = dirname(absolutePath); - const base = basename(absolutePath); - const tempPath = resolve(dir, `.${base}.tmp-${Date.now()}-${Math.random().toString(36).slice(2)}`); + timer = setTimeout(() => { + timedOut = true; + controller.abort(); + }, timeoutMs); + timer.unref?.(); try { - await writeFile(tempPath, content, "utf-8"); - await rename(tempPath, absolutePath); + const result = await action(controller.signal); + return { result }; } catch (error) { - try { - await unlink(tempPath); - } catch { - // ignore unlink error - } - throw error; - } -} - -// ───────────────────────────────────────────────────────────────────────────── -// #62 — init command -// ───────────────────────────────────────────────────────────────────────────── - -async function initCommand(filePath: string, options: CliOptions): Promise { - try { - const ext = extname(filePath).toLowerCase(); - if (ext !== ".json" && ext !== ".yaml" && ext !== ".yml") { - throw new Error( - `Unsupported file extension "${ext}". Supported extensions are .json, .yaml, and .yml.`, - ); + if (receivedSignal === "SIGINT") { + if (options.output === "json") { + console.error( + JSON.stringify({ + status: "cancelled", + error: { + code: "SIGINT", + message: "Execution interrupted by SIGINT.", + }, + }), + ); + } else { + const m = markers(options.ascii ?? false); + console.error(`\n${m.fail} Execution interrupted by SIGINT.`); + } + return { exitCode: 130 }; } - const baseName = basename(filePath); - const rawId = baseName.slice(0, baseName.length - ext.length); - let scenarioId = rawId - .toLowerCase() - .replace(/[^a-z0-9-]/g, "-") - .replace(/^-+|-+$/g, ""); - if (!scenarioId || !/^[a-z]/.test(scenarioId)) { - scenarioId = "custom-scenario"; - } - if (scenarioId.length > 64) { - scenarioId = scenarioId.slice(0, 64); + if (receivedSignal === "SIGTERM") { + if (options.output === "json") { + console.error( + JSON.stringify({ + status: "cancelled", + error: { + code: "SIGTERM", + message: "Execution terminated by SIGTERM.", + }, + }), + ); + } else { + const m = markers(options.ascii ?? false); + console.error(`\n${m.fail} Execution terminated by SIGTERM.`); + } + return { exitCode: 143 }; } - const definition = { - schemaVersion: 1, - id: scenarioId, - version: 1, - name: scenarioId.replace(/-/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()), - description: "Send XLM between two Testnet accounts and verify the balance update.", - network: "testnet", - accounts: [ - { id: "sender", generate: true, fund: true }, - { id: "receiver", generate: true, fund: true }, - ], - assets: [{ id: "xlm", type: "native" }], - steps: [ - { - id: "payment", - type: "payment", - from: "sender", - to: "receiver", - asset: "xlm", - amount: "10.0000000", - }, - ], - assertions: [ - { - type: "transactionConfirmed", - step: "payment", - }, - { - type: "balanceChangedBy", - account: "receiver", - asset: "xlm", - amount: "10.0000000", - }, - ], - }; - - // Canonical validator verification - validateScenarioDefinition(definition); - - let content: string; - if (ext === ".json") { - content = JSON.stringify(definition, null, 2) + "\n"; - } else { - content = stringifyYaml(definition); + if (timedOut) { + const timeoutError = { + code: "RUN_TIMEOUT", + message: `The run exceeded its ${timeoutMs}ms execution limit.`, + }; + if (options.output === "json") { + console.error( + JSON.stringify( + { + status: "error", + error: timeoutError, + }, + null, + 2, + ), + ); + } else { + const m = markers(options.ascii ?? false); + console.error(`\n${m.fail} Run timed out: ${timeoutError.message}`); + } + return { exitCode: 1 }; } - await writeArtifactFile(filePath, content, Boolean(options.overwrite), "scaffold"); - + // Normal execution error if (options.output === "json") { - console.log(JSON.stringify({ created: true, path: filePath, scenarioId }, null, 2)); - } else { - console.log(`Scenario scaffolded: ${filePath}`); - console.log("\nNext steps:"); - console.log(` Validate: esure validate ${filePath}`); - console.log(` Run: esure run ${filePath}\n`); - } - return 0; - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - if (options.output === "json") { - console.error(JSON.stringify({ error: { message: msg } })); + const errorObj: { message: string; report?: unknown } = { + message: error instanceof Error ? error.message : String(error), + }; + if (error && typeof error === "object" && "report" in error) { + errorObj.report = (error as { report: unknown }).report; + } + console.error(JSON.stringify({ status: "error", error: errorObj }, null, 2)); } else { - console.error(`Error: ${msg}`); + const m = markers(options.ascii ?? false); + console.error( + `\n${m.fail} Run failed: ${error instanceof Error ? error.message : String(error)}`, + ); } - return 1; + return { exitCode: 1 }; + } finally { + if (timer) clearTimeout(timer); + process.removeListener("SIGINT", onSigint); + process.removeListener("SIGTERM", onSigterm); } } @@ -617,88 +582,8 @@ async function validateCommand(filePath: string, options: CliOptions): Promise { - try { - const { size } = await handle.stat(); - if (size > MAX_REPORT_FILE_BYTES) { - throw new Error(`Report file exceeds the maximum size of ${MAX_REPORT_FILE_BYTES / (1024 * 1024)} MiB.`); - } - - const buffer = Buffer.alloc(MAX_REPORT_FILE_BYTES + 1); - let totalBytes = 0; - while (totalBytes < buffer.length) { - const { bytesRead } = await handle.read( - buffer, - totalBytes, - Math.min(64 * 1024, buffer.length - totalBytes), - totalBytes, - ); - if (bytesRead === 0) break; - totalBytes += bytesRead; - } - if (totalBytes > MAX_REPORT_FILE_BYTES) { - throw new Error(`Report file exceeds the maximum size of ${MAX_REPORT_FILE_BYTES / (1024 * 1024)} MiB.`); - } - - try { - return new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, totalBytes)); - } catch { - throw new Error("Report file is not valid UTF-8."); - } - } finally { - await handle.close(); - } -} - -async function validateReportCommand(filePath: string, options: CliOptions): Promise { - try { - if (extname(filePath).toLowerCase() !== ".json") { - throw new Error("Report file must have a .json extension."); - } - - const content = await readBoundedReportFile(filePath); - let parsed: unknown; - try { - parsed = JSON.parse(content); - } catch (error) { - const detail = error instanceof Error ? error.message : "invalid JSON"; - throw new Error(`Report file contains invalid JSON: ${detail}`); - } - - const report = validateRunReport(parsed); - if (options.output === "json") { - console.log(JSON.stringify({ - valid: true, - reportId: report.id, - scenarioId: report.scenarioId, - status: report.status, - }, null, 2)); - } else { - console.log("✓ Report is valid"); - console.log(` ID: ${report.id}`); - console.log(` Scenario: ${report.scenarioId}`); - console.log(` Status: ${report.status}`); - } - return 0; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - const issues = error instanceof RunReportValidationError ? error.issues : undefined; - if (options.output === "json") { - console.error(JSON.stringify({ valid: false, error: { message, ...(issues ? { issues } : {}) } }, null, 2)); - } else { - console.error(`✗ Report validation failed: ${message}`); - for (const issue of issues ?? []) console.error(` - ${issue}`); - } - return 1; - } -} - async function runCommand(target: string, options: CliOptions): Promise { - let scenario: ValidatedScenario | undefined; - let execution: LedgerExecution | undefined; - let report: ReturnType | undefined; - let executionError: Error | undefined; - + let scenario: ValidatedScenario; try { // Determine if target is a bundled scenario ID or file path if (isExplicitFile || target.includes("/") || target.includes("\\") || target.includes(".")) { @@ -713,41 +598,6 @@ async function runCommand(target: string, options: CliOptions): Promise } scenario = found; } - - // Execute scenario on Stellar Testnet - const gateway = new StellarTestnetGateway( - "https://horizon-testnet.stellar.org", - "https://friendbot.stellar.org", - ); - - if (options.output === "human") { - console.log(`Running scenario: ${scenario.name}`); - console.log(`ID: ${scenario.id} (v${scenario.version})\n`); - } - - // #66 — attach onStep / onAssertion for real-time progress in human mode - const execOptions = buildExecutionOptions(options); - execution = await gateway.execute(scenario, execOptions); - report = buildRunReport(scenario, execution); - - if (options.output === "json") { - console.log(JSON.stringify(report, null, 2)); - } else { - // In human mode the steps/assertions were already printed via callbacks. - // Only print the final summary — do not reprint the full step list. - const m = markers(options.ascii ?? false); - const passed = report.status === "passed"; - console.log(`\nResult: ${passed ? "PASSED" : "FAILED"}`); - console.log( - ` Steps: ${report.summary.stepsPassed}/${execution.steps.length} passed`, - ); - console.log( - ` Assertions: ${report.summary.assertionsPassed}/${execution.assertions.length} passed`, - ); - if (!passed) { - console.log(`\n${m.fail} One or more checks failed.`); - } - } } catch (error) { executionError = error instanceof Error ? error : new Error(String(error)); if (error && typeof error === "object" && "report" in error) { @@ -760,16 +610,9 @@ async function runCommand(target: string, options: CliOptions): Promise } if (options.output === "json") { - const errorObj: { message: string; report?: unknown; issues?: string[] } = { - message: error instanceof Error ? error.message : String(error), - }; - if (error && typeof error === "object" && "report" in error) { - errorObj.report = (error as { report: unknown }).report; - } - if (error && typeof error === "object" && "issues" in error && Array.isArray(error.issues)) { - errorObj.issues = error.issues as string[]; - } - console.error(JSON.stringify({ status: "error", error: errorObj }, null, 2)); + console.error( + JSON.stringify({ error: { message: error instanceof Error ? error.message : String(error) } }), + ); } else { const m = markers(options.ascii ?? false); console.error( @@ -783,91 +626,55 @@ async function runCommand(target: string, options: CliOptions): Promise } } - // Handle artifact exports (#59, #60, #157) - const scenarioId = scenario?.id ?? target; - let artifactError = false; + const effectiveTimeout = options.timeoutMs ?? DEFAULT_CLI_TIMEOUT_MS; - if (options.reportFile) { - try { - const reportPayload = report ?? { - scenarioId, - status: "error", - error: { message: executionError?.message ?? "Execution failed" }, - steps: execution?.steps ?? [], - assertions: execution?.assertions ?? [], - }; - await writeArtifactFile( - options.reportFile, - JSON.stringify(reportPayload, null, 2) + "\n", - Boolean(options.overwrite), - "report", + const executionResult = await runWithSignalsAndTimeout( + effectiveTimeout, + options, + async (signal) => { + // Execute scenario on Stellar Testnet + const gateway = new StellarTestnetGateway( + "https://horizon-testnet.stellar.org", + "https://friendbot.stellar.org", ); - } catch (err: unknown) { - artifactError = true; - const msg = err instanceof Error ? err.message : String(err); - if (options.output === "json") { - console.error(JSON.stringify({ error: { message: msg } })); - } else { - console.error(`Error writing report file: ${msg}`); - } - } - } - if (options.junit) { - try { - const xml = generateJunitXml([ - { - scenarioId, - execution, - error: executionError ? { message: executionError.message } : undefined, - }, - ]); - await writeArtifactFile( - options.junit, - xml, - Boolean(options.overwrite), - "JUnit", - ); - } catch (err: unknown) { - artifactError = true; - const msg = err instanceof Error ? err.message : String(err); - if (options.output === "json") { - console.error(JSON.stringify({ error: { message: msg } })); - } else { - console.error(`Error writing JUnit file: ${msg}`); + if (options.output === "human") { + console.log(`Running scenario: ${scenario.name}`); + console.log(`ID: ${scenario.id} (v${scenario.version})\n`); } - } + + const execOptions = buildExecutionOptions(options, signal); + return gateway.execute(scenario, execOptions); + }, + ); + + if (executionResult.exitCode !== undefined) { + return executionResult.exitCode; } - if (options.csv) { - try { - const csv = generateCsv([ - { - scenarioId, - execution, - error: executionError ? { message: executionError.message } : undefined, - }, - ]); - await writeArtifactFile( - options.csv, - csv, - Boolean(options.overwrite), - "CSV", - ); - } catch (err: unknown) { - artifactError = true; - const msg = err instanceof Error ? err.message : String(err); - if (options.output === "json") { - console.error(JSON.stringify({ error: { message: msg } })); - } else { - console.error(`Error writing CSV file: ${msg}`); - } + const execution = executionResult.result!; + const report = buildRunReport(scenario, execution); + + if (options.output === "json") { + console.log(JSON.stringify(report, null, 2)); + } else { + // In human mode the steps/assertions were already printed via callbacks. + // Only print the final summary — do not reprint the full step list. + const m = markers(options.ascii ?? false); + const passed = report.status === "passed"; + console.log(`\nResult: ${passed ? "PASSED" : "FAILED"}`); + console.log( + ` Steps: ${report.summary.stepsPassed}/${execution.steps.length} passed`, + ); + console.log( + ` Assertions: ${report.summary.assertionsPassed}/${execution.assertions.length} passed`, + ); + if (!passed) { + console.log(`\n${m.fail} One or more checks failed.`); } } - if (artifactError) return 1; - if (executionError) return 1; - return report?.status === "passed" ? 0 : 1; + return report.status === "passed" ? 0 : 1; } // ───────────────────────────────────────────────────────────────────────────── @@ -944,72 +751,89 @@ async function runSuiteCommand( return 1; } - // 2. Execute sequentially with isolated gateway per scenario - const results: SuiteScenarioResult[] = []; - const suiteJunitRuns: ScenarioRunResultForJunit[] = []; - let suitePassed = true; + // 2. Execute sequentially with isolated gateway per scenario and overall signal/timeout wiring + const effectiveTimeout = options.timeoutMs ?? DEFAULT_CLI_TIMEOUT_MS; - if (options.output === "human") { - console.log(`\nRun suite: ${ids.length} scenario(s)\n`); - } + const suiteRun = await runWithSignalsAndTimeout( + effectiveTimeout, + options, + async (signal) => { + const results: SuiteScenarioResult[] = []; + let suitePassed = true; - for (const scenario of scenarios) { - if (options.output === "human") { - console.log(`Running: ${scenario.id} — ${scenario.name}`); - } + if (options.output === "human") { + console.log(`\nRun suite: ${ids.length} scenario(s)\n`); + } - let result: SuiteScenarioResult; - let execution: LedgerExecution | undefined; - try { - const gateway = new StellarTestnetGateway( - "https://horizon-testnet.stellar.org", - "https://friendbot.stellar.org", - ); - // Per-scenario isolated execution with progress callbacks in human mode - const execOptions = buildExecutionOptions(options); - execution = await gateway.execute(scenario, execOptions); - const report = buildRunReport(scenario, execution); - result = { - scenarioId: scenario.id, - status: report.status, - summary: report.summary, - }; - suiteJunitRuns.push({ - scenarioId: scenario.id, - execution, - }); - } catch (error) { - const errObj = { message: error instanceof Error ? error.message : String(error) }; - result = { - scenarioId: scenario.id, - status: "error", - error: errObj, - }; - suiteJunitRuns.push({ - scenarioId: scenario.id, - error: errObj, - }); - } + for (const scenario of scenarios) { + if (signal.aborted) { + throw new SafeRunError({ + code: "RUN_TIMEOUT", + message: "The run was cancelled after reaching its execution limit.", + category: "timeout", + retryable: true, + }); + } - results.push(result); - if (result.status !== "passed") suitePassed = false; + if (options.output === "human") { + console.log(`Running: ${scenario.id} — ${scenario.name}`); + } - if (options.output === "human") { - const m = markers(options.ascii ?? false); - const marker = result.status === "passed" ? m.pass : m.fail; - if (result.status === "error") { - console.log(` ${marker} ${scenario.id}: error — ${result.error?.message ?? "unknown"}\n`); - } else { - const s = result.summary!; - console.log( - ` ${marker} ${scenario.id}: ${result.status} (steps ${s.stepsPassed}/${s.stepsPassed + s.stepsFailed}, checks ${s.assertionsPassed}/${s.assertionsPassed + s.assertionsFailed})\n`, - ); + let result: SuiteScenarioResult; + try { + const gateway = new StellarTestnetGateway( + "https://horizon-testnet.stellar.org", + "https://friendbot.stellar.org", + ); + // Per-scenario isolated execution with progress callbacks in human mode + const execOptions = buildExecutionOptions(options, signal); + const execution = await gateway.execute(scenario, execOptions); + const report = buildRunReport(scenario, execution); + result = { + scenarioId: scenario.id, + status: report.status, + summary: report.summary, + }; + } catch (error) { + if (signal.aborted) { + throw error; + } + result = { + scenarioId: scenario.id, + status: "error", + error: { message: error instanceof Error ? error.message : String(error) }, + }; + } + + results.push(result); + if (result.status !== "passed") suitePassed = false; + + if (options.output === "human") { + const m = markers(options.ascii ?? false); + const marker = result.status === "passed" ? m.pass : m.fail; + if (result.status === "error") { + console.log(` ${marker} ${scenario.id}: error — ${result.error?.message ?? "unknown"}\n`); + } else { + const s = result.summary!; + console.log( + ` ${marker} ${scenario.id}: ${result.status} (steps ${s.stepsPassed}/${s.stepsPassed + s.stepsFailed}, checks ${s.assertionsPassed}/${s.assertionsPassed + s.assertionsFailed})\n`, + ); + } + } + + if (!suitePassed && failFast) break; } - } - if (!suitePassed && failFast) break; + return { results, suitePassed }; + }, + ); + + if (suiteRun.exitCode !== undefined) { + return suiteRun.exitCode; } + const { results, suitePassed } = suiteRun.result!; + // 3. Output final summary const passedCount = results.filter((r) => r.status === "passed").length; const failedCount = results.length - passedCount; @@ -1256,7 +1080,7 @@ async function main(): Promise { const { command, args, options, failFast, parseError } = parseCliArgs(); if (parseError) { - console.error(`Error: ${parseError}`); + reportArgumentError(parseError.message, parseError.usage, options.output); return 1; } @@ -1280,8 +1104,11 @@ async function main(): Promise { return listCommand(options); case "validate": if (args.length === 0) { - console.error("Error: Missing file path argument"); - console.error("Usage: esure validate "); + reportArgumentError( + "Missing file path argument", + "Usage: esure validate ", + options.output, + ); return 1; } if (args.length > 1) { @@ -1318,8 +1145,11 @@ async function main(): Promise { return runCommand(options.file, options, true); } if (args.length === 0) { - console.error("Error: Missing scenario ID or file path"); - console.error("Usage: esure run or esure run "); + reportArgumentError( + "Missing scenario ID or file path", + "Usage: esure run or esure run ", + options.output, + ); return 1; } if (args.length > 1) { @@ -1330,8 +1160,11 @@ async function main(): Promise { return runCommand(args[0], options, false); case "run-suite": if (args.length === 0) { - console.error("Error: Missing scenario IDs"); - console.error("Usage: esure run-suite [...]"); + reportArgumentError( + "Missing scenario IDs", + "Usage: esure run-suite [...]", + options.output, + ); return 1; } return runSuiteCommand(args, options, failFast); @@ -1345,8 +1178,11 @@ async function main(): Promise { case "doctor": return doctorCommand(options); default: - console.error(`Error: Unknown command "${command}"`); - console.error('Run "esure --help" for usage information'); + reportArgumentError( + `Unknown command "${command}"`, + 'Run "esure --help" for usage information', + options.output, + ); return 1; } } diff --git a/esure-cli/test/cli.test.ts b/esure-cli/test/cli.test.ts index 1fe4fa5..9ba4d8c 100644 --- a/esure-cli/test/cli.test.ts +++ b/esure-cli/test/cli.test.ts @@ -211,9 +211,10 @@ describe("CLI commands", () => { status: "passed", summary: { stepsPassed: 1, stepsFailed: 0, assertionsPassed: 1, assertionsFailed: 0 }, }); - expect(mocks.execute).toHaveBeenCalledWith(expect.objectContaining({ id: scenario.id }), { - stepTimeoutMs: 30000, - }); + expect(mocks.execute).toHaveBeenCalledWith( + expect.objectContaining({ id: scenario.id }), + expect.objectContaining({ stepTimeoutMs: 30000 }), + ); }); it("runs a scenario file and prints a human-readable passing report", async () => { @@ -726,3 +727,282 @@ describe("#68 ASCII status markers", () => { }); }); +// ───────────────────────────────────────────────────────────────────────────── +// #54 — Structured JSON argument errors +// ───────────────────────────────────────────────────────────────────────────── + +describe("#54 CLI argument errors conform to JSON mode", () => { + beforeEach(() => { + stdout = []; + stderr = []; + originalArgv = process.argv; + originalExitCode = process.exitCode; + process.exitCode = undefined; + vi.spyOn(console, "log").mockImplementation((...values: unknown[]) => { + stdout.push(values.join(" ")); + }); + vi.spyOn(console, "error").mockImplementation((...values: unknown[]) => { + stderr.push(values.join(" ")); + }); + mocks.list.mockReturnValue(catalog); + mocks.find.mockImplementation((id: string) => id === scenario.id ? structuredClone(scenario) : undefined); + mocks.createRegistry.mockReturnValue({ list: mocks.list, find: mocks.find }); + mocks.execute.mockResolvedValue(passedExecution); + }); + + afterEach(() => { + process.argv = originalArgv; + process.exitCode = originalExitCode; + vi.restoreAllMocks(); + }); + + it("returns JSON error envelope for unknown command with --output json", async () => { + expect(await invokeCli("unknown-cmd", "--output", "json")).toBe(1); + expect(stdout).toEqual([]); + const err = JSON.parse(stderr.join("\n")); + expect(err).toEqual({ + error: { + code: "INVALID_ARGUMENT", + message: 'Unknown command "unknown-cmd"', + }, + }); + }); + + it("returns JSON error envelope for missing validate file target with -o json", async () => { + expect(await invokeCli("validate", "-o", "json")).toBe(1); + expect(stdout).toEqual([]); + const err = JSON.parse(stderr.join("\n")); + expect(err).toEqual({ + error: { + code: "INVALID_ARGUMENT", + message: "Missing file path argument", + }, + }); + }); + + it("returns JSON error envelope for missing run scenario target with --output json", async () => { + expect(await invokeCli("run", "--output", "json")).toBe(1); + expect(stdout).toEqual([]); + const err = JSON.parse(stderr.join("\n")); + expect(err).toEqual({ + error: { + code: "INVALID_ARGUMENT", + message: "Missing scenario ID or file path", + }, + }); + }); + + it("returns JSON error envelope for missing run-suite target with --output json", async () => { + expect(await invokeCli("run-suite", "--output", "json")).toBe(1); + expect(stdout).toEqual([]); + const err = JSON.parse(stderr.join("\n")); + expect(err).toEqual({ + error: { + code: "INVALID_ARGUMENT", + message: "Missing scenario IDs", + }, + }); + }); + + it("returns JSON error envelope for invalid output format with --output=unknown", async () => { + expect(await invokeCli("list", "--output", "xml")).toBe(1); + expect(stdout).toEqual([]); + expect(stderr.join("\n")).toContain("Invalid output format"); + }); + + it("returns JSON error envelope for unknown option when --output json is specified", async () => { + expect(await invokeCli("list", "--bogus-flag", "--output", "json")).toBe(1); + expect(stdout).toEqual([]); + const err = JSON.parse(stderr.join("\n")); + expect(err.error.code).toBe("INVALID_ARGUMENT"); + expect(err.error.message).toContain("Unknown option"); + }); +}); + +// ───────────────────────────────────────────────────────────────────────────── +// #55 — Bounded overall run timeout +// ───────────────────────────────────────────────────────────────────────────── + +describe("#55 bounded overall run timeout", () => { + beforeEach(() => { + stdout = []; + stderr = []; + originalArgv = process.argv; + originalExitCode = process.exitCode; + process.exitCode = undefined; + vi.spyOn(console, "log").mockImplementation((...values: unknown[]) => { + stdout.push(values.join(" ")); + }); + vi.spyOn(console, "error").mockImplementation((...values: unknown[]) => { + stderr.push(values.join(" ")); + }); + mocks.list.mockReturnValue(catalog); + mocks.find.mockImplementation((id: string) => id === scenario.id ? structuredClone(scenario) : undefined); + mocks.createRegistry.mockReturnValue({ list: mocks.list, find: mocks.find }); + }); + + afterEach(() => { + process.argv = originalArgv; + process.exitCode = originalExitCode; + vi.restoreAllMocks(); + }); + + it("rejects non-integer or out-of-range timeout values in JSON mode", async () => { + expect(await invokeCli("run", scenario.id, "--timeout-ms", "0", "--output", "json")).toBe(1); + const err = JSON.parse(stderr.join("\n")); + expect(err.error.code).toBe("INVALID_ARGUMENT"); + expect(err.error.message).toContain("--timeout-ms must be an integer between 1 and 600000"); + + stderr = []; + expect(await invokeCli("run", scenario.id, "--timeout-ms", "abc", "--output", "json")).toBe(1); + const err2 = JSON.parse(stderr.join("\n")); + expect(err2.error.code).toBe("INVALID_ARGUMENT"); + expect(err2.error.message).toContain("--timeout-ms must be an integer between 1 and 600000"); + }); + + it("aborts execution and returns stable timeout error in JSON mode", async () => { + mocks.execute.mockImplementation((_sc: unknown, opts: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + opts.signal?.addEventListener("abort", () => { + reject(new Error("aborted")); + }); + }); + }); + + const exitCode = await invokeCli("run", scenario.id, "--timeout-ms", "10", "--output", "json"); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(process.exitCode).toBe(1); + const err = JSON.parse(stderr.join("\n")); + expect(err.status).toBe("error"); + expect(err.error.code).toBe("RUN_TIMEOUT"); + expect(err.error.message).toContain("10ms"); + }); + + it("aborts execution and reports timeout in human mode", async () => { + mocks.execute.mockImplementation((_sc: unknown, opts: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + opts.signal?.addEventListener("abort", () => { + reject(new Error("aborted")); + }); + }); + }); + + await invokeCli("run", scenario.id, "--timeout-ms", "10"); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(process.exitCode).toBe(1); + expect(stderr.join("\n")).toContain("Run timed out: The run exceeded its 10ms execution limit."); + }); + + it("clears timeout handle when execution completes before deadline", async () => { + mocks.execute.mockResolvedValueOnce(passedExecution); + expect(await invokeCli("run", scenario.id, "--timeout-ms", "10000", "--output", "json")).toBe(0); + const out = JSON.parse(stdout.join("\n")); + expect(out.status).toBe("passed"); + }); +}); + +// ───────────────────────────────────────────────────────────────────────────── +// #56 — SIGINT and SIGTERM graceful cancellation +// ───────────────────────────────────────────────────────────────────────────── + +describe("#56 graceful SIGINT and SIGTERM cancellation", () => { + beforeEach(() => { + stdout = []; + stderr = []; + originalArgv = process.argv; + originalExitCode = process.exitCode; + process.exitCode = undefined; + vi.spyOn(console, "log").mockImplementation((...values: unknown[]) => { + stdout.push(values.join(" ")); + }); + vi.spyOn(console, "error").mockImplementation((...values: unknown[]) => { + stderr.push(values.join(" ")); + }); + mocks.list.mockReturnValue(catalog); + mocks.find.mockImplementation((id: string) => id === scenario.id ? structuredClone(scenario) : undefined); + mocks.createRegistry.mockReturnValue({ list: mocks.list, find: mocks.find }); + }); + + afterEach(() => { + process.argv = originalArgv; + process.exitCode = originalExitCode; + vi.restoreAllMocks(); + }); + + it("handles SIGINT with exit code 130 and concise JSON output", async () => { + const initialSigintListeners = process.listenerCount("SIGINT"); + const initialSigtermListeners = process.listenerCount("SIGTERM"); + + mocks.execute.mockImplementation((_sc: unknown, opts: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + opts.signal?.addEventListener("abort", () => { + reject(new Error("operation aborted")); + }); + setImmediate(() => { + process.emit("SIGINT"); + }); + }); + }); + + const exitCode = await invokeCli("run", scenario.id, "--output", "json"); + expect(exitCode).toBe(130); + const err = JSON.parse(stderr.join("\n")); + expect(err).toEqual({ + status: "cancelled", + error: { + code: "SIGINT", + message: "Execution interrupted by SIGINT.", + }, + }); + + // Verify listeners are cleaned up + expect(process.listenerCount("SIGINT")).toBe(initialSigintListeners); + expect(process.listenerCount("SIGTERM")).toBe(initialSigtermListeners); + }); + + it("handles SIGTERM with exit code 143 and concise human output", async () => { + mocks.execute.mockImplementation((_sc: unknown, opts: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + opts.signal?.addEventListener("abort", () => { + reject(new Error("operation aborted")); + }); + setImmediate(() => { + process.emit("SIGTERM"); + }); + }); + }); + + const exitCode = await invokeCli("run", scenario.id); + expect(exitCode).toBe(143); + expect(stderr.join("\n")).toContain("Execution terminated by SIGTERM."); + }); +}); + +// ───────────────────────────────────────────────────────────────────────────── +// #57 — Dynamic version from package metadata +// ───────────────────────────────────────────────────────────────────────────── + +describe("#57 derive --version from package metadata", () => { + beforeEach(() => { + stdout = []; + stderr = []; + originalArgv = process.argv; + originalExitCode = process.exitCode; + process.exitCode = undefined; + vi.spyOn(console, "log").mockImplementation((...values: unknown[]) => { + stdout.push(values.join(" ")); + }); + }); + + afterEach(() => { + process.argv = originalArgv; + process.exitCode = originalExitCode; + vi.restoreAllMocks(); + }); + + it("displays version matching package.json", async () => { + expect(await invokeCli("--version")).toBe(0); + expect(stdout.join("\n")).toBe("esure v0.1.0"); + }); +}); +