diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 0b0564b97..20bbe65de 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -1,4 +1,4 @@ { "printWidth": 80, - "ignorePatterns": ["dist/", "node_modules/"] + "ignorePatterns": ["dist/", "node_modules/", "e2e/fixtures/flows/"] } diff --git a/.oxlintrc.json b/.oxlintrc.json index d9083ebc0..fd24c147d 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -102,8 +102,33 @@ "options": { "typeAware": true }, - "ignorePatterns": ["dist/", "node_modules/", "knip.config.ts"], + "ignorePatterns": [ + "dist/", + "node_modules/", + "knip.config.ts", + "e2e/fixtures/flows/" + ], "overrides": [ + { + "files": ["e2e/**/*.ts"], + "rules": { + "eslint/no-restricted-imports": [ + "error", + { + "patterns": [ + { + "group": ["~/*", "~/**"], + "message": "e2e/ must not import from src/ — it treats the CLI as a subprocess black box." + }, + { + "group": ["bun", "bun:*"], + "message": "e2e/ harness uses node: builtins for portability — avoid Bun module imports." + } + ] + } + ] + } + }, { "files": ["src/core/**/*.ts"], "rules": { diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 000000000..06da97196 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,196 @@ +# e2e — isolated repo-readiness E2E framework + +A manually-run, high-level end-to-end harness that drives the **real built CLI** +across a data-driven matrix of repo shapes. It exists to prove `qawolf flows run` +works on every project/monorepo layout, on **both** the Node and compiled-binary +channels, with zero project pollution — repeatably and with a clean report. + +It is **not** part of the product and **not** a constant CI gate. Run it by hand +when you want to verify that runtime-deps resolution still holds across shapes. + +## What it is & why it's isolated + +The harness treats the CLI as a **subprocess black box**. It imports **nothing** +from `src/` — it builds the CLI, then spawns the built artifacts and asserts on +exit code, JUnit output, and on-disk side effects. + +Isolation guarantees: + +- **Run by reference, no global mutation.** It runs `bun run build:binary` once + (which produces `dist/cli.js` _and_ `dist/qawolf` in one pass), then spawns those + artifacts directly — `node dist/cli.js …` for the node channel, `dist/qawolf …` + for the binary channel. No global install, no `npm link`, no `~/.local/bin` + changes. +- **Isolated managed runtime.** Each run points `QAWOLF_RUNTIME_DIR` at a throwaway + tmp dir, so it never touches the real `~/Library/Application Support/qawolf-nodejs`. + The dir is shared across all cases in a run so the runtime + browser download + warms once per channel (the managed runtime keys node vs binary by hash + internally), then is removed on cleanup. +- **Throwaway project dirs.** Every case materializes its shape into a fresh + `mkdtemp` project dir and removes it afterward (unless `--no-cleanup`). +- **No `src/` coupling.** `report.ts` uses `@clack/prompts` directly rather than + `src/shell/ui`. The CLI is only ever a spawned process. + +## How to run + +```bash +bun run e2e -- repo-readiness # full matrix, both channels +bun run e2e -- repo-readiness --channel node # node channel only +bun run e2e -- repo-readiness --channel binary # binary channel only +bun run e2e -- repo-readiness --no-cleanup # keep tmp dirs (prints retained paths) +bun run e2e -- repo-readiness --json # structured output (also the non-TTY fallback) +``` + +There is a **single** `e2e` script by design — the suite is a positional arg, not a +per-suite script. You can equivalently run `bun e2e/run.ts repo-readiness`. Omitting +the suite name runs every registered suite. + +The first positional that doesn't start with `--` is the suite name; `--channel` +takes `node`, `binary`, or `both` (default `both`). `--no-cleanup` prints the +retained workspace path in each cell's output for debugging. `--json` (or any +non-TTY stdout) emits `{ results, exitCode }` instead of the Clack report. + +> **First run is slow.** Each channel builds its artifact and warms the managed +> runtime + browser exactly once. Subsequent runs reuse the build and are fast. + +The process exits non-zero if any cell fails. + +## The repo-shape matrix + +Eight cases × two channels = 16 cells. Cases 01–07 use the simple flow +(`simpleNav` — navigates `example.com`, no QA Wolf account needed). Case 08 is the +hard one. + +| # | Shape | Proves | +| --- | -------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| 01 | flow only, **no `package.json`** | runs with nothing to resolve | +| 02 | npm single-package (ESM) | baseline | +| 03 | single + `bun.lock` | bun single | +| 04 | npm workspace, flow in **leaf, no leaf `node_modules`** | originally-reported monorepo failure | +| 05 | pnpm workspace (`pnpm-workspace.yaml` + lock) | pnpm workspace | +| 06 | yarn workspace (`workspaces` + `yarn.lock`) | yarn workspace | +| 07 | bun workspace (`workspaces` + `bun.lock`) | bun workspace | +| 08 | declares `diff@^8.0.3` + `sharp`, flow imports `FILE_HEADERS_ONLY` from `diff`, runs `sharp` | inner-hop version-shadowing fix **and** binary native-module load | + +Per-case assertions (all must hold, see `harness/assertions.ts`): + +1. **CLI exits 0** and JUnit reports `failures="0"` and `errors="0"` with at least + one test (a missing JUnit file, or `tests="0"`, is also a failure — it means the + flow never executed). +2. **Zero project pollution** — no `node_modules` dir written into the project tree + outside the isolated `.qawolf/` cache. +3. **Case 08 only** — output/JUnit contains **none** of the forbidden regression + strings: `FILE_HEADERS_ONLY`, `Could not load the "sharp" module`, + `Cannot find package`. + +## How to add a CASE + +A case is a `RepoShape` pushed onto `repoShapes` in `fixtures/shapes.ts`. If it +needs new file bodies, add them as content consts in `fixtures/shapeFiles.ts` +first (verbatim, byte-for-byte) and import them. + +The `RepoShape` shape (`harness/types.ts`): + +```ts +type RepoShape = { + readonly name: string; // e.g. "09-deno-single" — also keys case-08 assertions ("08"/"native") + readonly proves: string; // one-line description of what this shape exercises + readonly files: readonly ShapeFile[]; // { path, content }[] — excludes the flow itself + readonly flow: FlowTemplate; // "simpleNav" | "nativeAndVersioned" + readonly runDir: string; // subdir to run `flows run` from ("" = project root) + readonly flowArg: string; // flow path relative to runDir, passed to `flows run` +}; +``` + +The harness writes each `files` entry, then writes the named flow template at +`join(runDir, flowArg)`, then spawns `flows run --junit ` with +`cwd = join(projectDir, runDir)`. Concrete example: + +```ts +{ + name: "02-npm-single", + proves: "npm single-package (ESM) — baseline", + files: [{ path: "package.json", content: npmSinglePackageJson }], + flow: "simpleNav", + runDir: "", + flowArg: "src/flows/smoke/compat-smoke.flow.ts", +} +``` + +For a workspace shape, point `runDir` at the leaf (e.g. `"packages/app"`) and add +the root + leaf manifests to `files`. + +## How to add a SUITE + +Adding a suite is a **one-file change plus one registry line**. + +1. Drop `suites/.ts` exporting a `Suite`: + +```ts +// suites/smokeOnly.ts +import { repoShapes } from "../fixtures/shapes.js"; +import type { Suite } from "../harness/types.js"; + +/** Smoke-only: the first two shapes, node channel only. */ +export const smokeOnlySuite: Suite = { + name: "smoke-only", + cases: repoShapes.slice(0, 2), + channels: ["node"], +}; +``` + +1. Register it in `suites/index.ts`: + +```ts +import { smokeOnlySuite } from "./smokeOnly.js"; + +const suites: Record = { + [repoReadinessSuite.name]: repoReadinessSuite, + [smokeOnlySuite.name]: smokeOnlySuite, +}; +``` + +Run it with `bun run e2e -- smoke-only`. No harness changes — `run.ts` resolves any +registered suite by name, builds only the channels the suite declares, and reports. + +## Layout + +```text +e2e/ +├── README.md +├── run.ts # entry: parse args, resolve suite(s), run case × channel, report, exit +├── harness/ +│ ├── types.ts # Channel, RepoShape, Suite, CaseResult, ShapeFile, FlowTemplate +│ ├── channels.ts # resolveChannels(): build once → run-by-reference channels +│ ├── tmpWorkspace.ts # createRuntimeRoot() + createTmpProject(): isolated dirs + QAWOLF_RUNTIME_DIR +│ ├── materialize.ts # write a RepoShape (files + flow template) into a tmp dir +│ ├── spawnCli.ts # node:child_process spawn → { exitCode, stdout, stderr } (never rejects) +│ ├── runCase.ts # spawn the CLI on one case/channel → CaseResult +│ ├── assertions.ts # parseJunit, assertExitAndJunit, scanPollution, assertCase08Strings +│ └── report.ts # @clack/prompts report (or --json/non-TTY); returns exit code +├── fixtures/ +│ ├── flows/ # opaque flow template assets (see note below) +│ │ ├── simpleNav.flow.ts +│ │ └── nativeAndVersioned.flow.ts +│ ├── shapes.ts # repoShapes: the 8 RepoShape builders +│ └── shapeFiles.ts # verbatim project-file bodies (package.json, lockfile stubs) +└── suites/ + ├── index.ts # registry: getSuite(name), allSuites() + └── repoReadiness.ts # first suite: 8 cases, [node, binary] +``` + +## Note on flow templates + +`fixtures/flows/*.flow.ts` are **opaque fixture assets**. The harness reads them as +**text** and writes them into the tmp project; the CLI subprocess is what actually +executes them. They import dependencies this repo does not have (`sharp`, `diff`, +`@qawolf/flows`), so they are deliberately excluded from this repo's tooling: + +- `tsconfig.json` — `exclude: ["…", "e2e/fixtures/flows"]` +- `.oxlintrc.json` — `ignorePatterns: ["…", "e2e/fixtures/flows/"]` +- `.oxfmtrc.json` — `ignorePatterns: ["…", "e2e/fixtures/flows/"]` +- `knip.config.ts` — `ignore: ["e2e/fixtures/flows/**"]` (they're never imported) + +The rest of `e2e/` **is** covered by typecheck, lint, format, and knip. Keep each +module under 150 lines and use `node:` builtins (the `Bun` global is banned by +oxlint in this tree). diff --git a/e2e/fixtures/flows/nativeAndVersioned.flow.ts b/e2e/fixtures/flows/nativeAndVersioned.flow.ts new file mode 100644 index 000000000..59092bb00 --- /dev/null +++ b/e2e/fixtures/flows/nativeAndVersioned.flow.ts @@ -0,0 +1,24 @@ +import { flow } from "@qawolf/flows/web"; +import sharp from "sharp"; +import { FILE_HEADERS_ONLY } from "diff"; + +export default flow( + "Native + Versioned Deps", + { target: "Web - Chrome", launch: {} }, + async ({ page, test }) => { + await test("project deps + native module resolve", async () => { + // diff@^8.0.3 only: FILE_HEADERS_ONLY does not exist in the executor's diff@8.0.2. + if (FILE_HEADERS_ONLY === undefined) { + throw new Error("diff FILE_HEADERS_ONLY missing — wrong diff version resolved"); + } + // sharp's native addon must actually load and run. + const png = await sharp({ + create: { width: 4, height: 4, channels: 3, background: { r: 1, g: 2, b: 3 } }, + }) + .png() + .toBuffer(); + if (png.length === 0) throw new Error("sharp produced no output"); + await page.goto("https://example.com"); + }); + }, +); diff --git a/e2e/fixtures/flows/simpleNav.flow.ts b/e2e/fixtures/flows/simpleNav.flow.ts new file mode 100644 index 000000000..0b45f28c7 --- /dev/null +++ b/e2e/fixtures/flows/simpleNav.flow.ts @@ -0,0 +1,11 @@ +import { flow } from "@qawolf/flows/web"; + +export default flow( + "Compat Smoke", + { target: "Web - Chrome", launch: {} }, + async ({ page, test }) => { + await test("navigates", async () => { + await page.goto("https://example.com"); + }); + }, +); diff --git a/e2e/fixtures/shapeFiles.ts b/e2e/fixtures/shapeFiles.ts new file mode 100644 index 000000000..88a7a76b2 --- /dev/null +++ b/e2e/fixtures/shapeFiles.ts @@ -0,0 +1,43 @@ +// Verbatim project-file bodies ported from cli-test/ cases 01–08. Each string +// reproduces the on-disk file byte-for-byte (trailing newline included) so a +// materialized shape matches the cli-test tree exactly. + +export const npmSinglePackageJson = `{ "name": "npm-single", "type": "module", "version": "1.0.0" }\n`; + +export const bunSinglePackageJson = `{ "name": "bun-single", "type": "module", "version": "1.0.0" }\n`; + +// Shared by the bun single-package (03) and bun workspace (07) shapes. +export const bunLock = `{ + "lockfileVersion": 1, + "workspaces": {}, + "packages": {} +} +`; + +export const nativeVersionedPackageJson = `{ + "name": "native-versioned", + "type": "module", + "version": "1.0.0", + "dependencies": { + "diff": "^8.0.3", + "sharp": "0.34.3" + } +} +`; + +export const npmWorkspaceRootPackageJson = `{ "name": "npm-ws-root", "private": true, "workspaces": ["packages/*"] }\n`; + +export const yarnWorkspaceRootPackageJson = `{ "name": "yarn-ws-root", "private": true, "workspaces": ["packages/*"] }\n`; + +export const bunWorkspaceRootPackageJson = `{ "name": "bun-ws-root", "private": true, "workspaces": ["packages/*"] }\n`; + +export const pnpmWorkspaceRootPackageJson = `{ "name": "pnpm-ws-root", "private": true }\n`; + +export const pnpmWorkspaceYaml = `packages:\n - "packages/*"\n`; + +export const pnpmLock = `lockfileVersion: '9.0'\n`; + +export const yarnLock = `# THIS IS AN AUTOGENERATED FILE.\n__metadata:\n version: 8\n`; + +// Shared leaf-package manifest for every workspace shape (04–07). +export const workspaceAppPackageJson = `{ "name": "@ws/app", "type": "module", "version": "1.0.0" }\n`; diff --git a/e2e/fixtures/shapes.ts b/e2e/fixtures/shapes.ts new file mode 100644 index 000000000..ce33708a1 --- /dev/null +++ b/e2e/fixtures/shapes.ts @@ -0,0 +1,106 @@ +import type { RepoShape } from "../harness/types.js"; +import { + bunLock, + bunSinglePackageJson, + bunWorkspaceRootPackageJson, + nativeVersionedPackageJson, + npmSinglePackageJson, + npmWorkspaceRootPackageJson, + pnpmLock, + pnpmWorkspaceRootPackageJson, + pnpmWorkspaceYaml, + workspaceAppPackageJson, + yarnLock, + yarnWorkspaceRootPackageJson, +} from "./shapeFiles.js"; + +// Every shape runs the same flow path relative to its run dir. +const flowArg = "src/flows/smoke/compat-smoke.flow.ts"; + +export const repoShapes: RepoShape[] = [ + { + name: "01-empty", + proves: "flow only, no package.json — runs with nothing to resolve", + files: [], + flow: "simpleNav", + runDir: "", + flowArg, + }, + { + name: "02-npm-single", + proves: "npm single-package (ESM) — baseline", + files: [{ path: "package.json", content: npmSinglePackageJson }], + flow: "simpleNav", + runDir: "", + flowArg, + }, + { + name: "03-bun-single", + proves: "single package + bun.lock — bun single", + files: [ + { path: "package.json", content: bunSinglePackageJson }, + { path: "bun.lock", content: bunLock }, + ], + flow: "simpleNav", + runDir: "", + flowArg, + }, + { + name: "04-npm-workspace", + proves: + "npm workspace, flow in leaf with no leaf node_modules — originally-reported monorepo failure", + files: [ + { path: "package.json", content: npmWorkspaceRootPackageJson }, + { path: "packages/app/package.json", content: workspaceAppPackageJson }, + ], + flow: "simpleNav", + runDir: "packages/app", + flowArg, + }, + { + name: "05-pnpm-workspace", + proves: "pnpm workspace (pnpm-workspace.yaml + lock) — pnpm workspace", + files: [ + { path: "package.json", content: pnpmWorkspaceRootPackageJson }, + { path: "pnpm-workspace.yaml", content: pnpmWorkspaceYaml }, + { path: "pnpm-lock.yaml", content: pnpmLock }, + { path: "packages/app/package.json", content: workspaceAppPackageJson }, + ], + flow: "simpleNav", + runDir: "packages/app", + flowArg, + }, + { + name: "06-yarn-workspace", + proves: "yarn workspace (workspaces + yarn.lock) — yarn workspace", + files: [ + { path: "package.json", content: yarnWorkspaceRootPackageJson }, + { path: "yarn.lock", content: yarnLock }, + { path: "packages/app/package.json", content: workspaceAppPackageJson }, + ], + flow: "simpleNav", + runDir: "packages/app", + flowArg, + }, + { + name: "07-bun-workspace", + proves: "bun workspace (workspaces + bun.lock) — bun workspace", + files: [ + { path: "package.json", content: bunWorkspaceRootPackageJson }, + { path: "bun.lock", content: bunLock }, + { path: "packages/app/package.json", content: workspaceAppPackageJson }, + ], + flow: "simpleNav", + runDir: "packages/app", + flowArg, + }, + { + name: "08-native-and-versioned-deps", + proves: + "declares diff@^8.0.3 + sharp; flow imports FILE_HEADERS_ONLY and runs sharp — inner-hop version-shadowing fix and binary native-module load", + files: [{ path: "package.json", content: nativeVersionedPackageJson }], + flow: "nativeAndVersioned", + runDir: "", + flowArg, + }, +]; diff --git a/e2e/harness/assertions.ts b/e2e/harness/assertions.ts new file mode 100644 index 000000000..26c23d42b --- /dev/null +++ b/e2e/harness/assertions.ts @@ -0,0 +1,90 @@ +import { join } from "node:path"; + +import { globSync } from "tinyglobby"; + +/** + * Strings that must NEVER appear in case-08 output — each is a regression of the + * inner-hop version-shadowing fix or the binary native-module load. + */ +const case08ForbiddenStrings = [ + "FILE_HEADERS_ONLY", + 'Could not load the "sharp" module', + "Cannot find package", +] as const; + +export type JunitSummary = { + readonly tests: number; + readonly failures: number; + readonly errors: number; +}; + +/** Parses the root `testsuites` counts; undefined when no JUnit was produced. */ +export function parseJunit(xml: string): JunitSummary | undefined { + const tests = matchCount(xml, "tests"); + if (tests === undefined) return undefined; + return { + tests, + failures: matchCount(xml, "failures") ?? 0, + errors: matchCount(xml, "errors") ?? 0, + }; +} + +function matchCount(xml: string, attr: string): number | undefined { + const value = xml.match(new RegExp(`${attr}="(\\d+)"`))?.[1]; + return value === undefined ? undefined : Number(value); +} + +/** + * Reasons the CLI run itself failed: non-zero exit, missing JUnit, zero tests + * (the flow never executed — guards against a silent green), or any + * failures/errors. + */ +export function assertExitAndJunit( + exitCode: number, + junit: JunitSummary | undefined, +): string[] { + const reasons: string[] = []; + if (exitCode !== 0) reasons.push(`CLI exited ${exitCode}, expected 0`); + if (junit === undefined) { + reasons.push("no JUnit output produced"); + return reasons; + } + if (junit.tests < 1) + reasons.push("JUnit reported 0 tests — flow did not run"); + if (junit.failures > 0) + reasons.push(`JUnit reported ${junit.failures} failure(s)`); + if (junit.errors > 0) reasons.push(`JUnit reported ${junit.errors} error(s)`); + return reasons; +} + +/** + * Mirrors the run-all.sh `find` for node_modules outside the .qawolf cache: any + * node_modules dir written into the project (outside the isolated .qawolf cache) + * is pollution. + */ +export function scanPollution(projectDir: string): string[] { + const matches = globSync("**/node_modules", { + cwd: projectDir, + onlyDirectories: true, + dot: true, + ignore: ["**/.qawolf/**"], + }); + return matches.map((match) => join(projectDir, match)); +} + +/** + * For the native+versioned case only, fails if output contains any forbidden + * regression string. Keyed off the shape name ("08" / "native"); other cases + * return no reasons. + */ +export function assertCase08Strings( + shapeName: string, + output: string, +): string[] { + const isNativeVersionedCase = + shapeName.includes("08") || shapeName.includes("native"); + if (!isNativeVersionedCase) return []; + return case08ForbiddenStrings + .filter((forbidden) => output.includes(forbidden)) + .map((forbidden) => `output contains forbidden string: ${forbidden}`); +} diff --git a/e2e/harness/channels.ts b/e2e/harness/channels.ts new file mode 100644 index 000000000..e57bb898b --- /dev/null +++ b/e2e/harness/channels.ts @@ -0,0 +1,49 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; + +import { spawnCli } from "./spawnCli.js"; +import type { Channel, ChannelName } from "./types.js"; + +/** + * Builds both CLI artifacts once and returns run-by-reference channels. The + * `build:binary` script also runs `build`, producing dist/cli.js AND + * dist/qawolf in a single pass, so only one build runs regardless of which + * channels are requested. The harness runs under bun, so the node channel + * spawns the real `node` binary (not process.execPath, which is bun). + */ +export async function resolveChannels( + channelNames: readonly ChannelName[], +): Promise { + const repoRoot = process.cwd(); + const result = await spawnCli("bun", ["run", "build:binary"], { + cwd: repoRoot, + env: process.env, + }); + if (result.exitCode !== 0) { + throw new Error( + `build:binary failed (exit ${result.exitCode}):\n${result.stderr}`, + ); + } + const absCliJs = join(repoRoot, "dist", "cli.js"); + const absQawolf = join(repoRoot, "dist", "qawolf"); + assertArtifact(absCliJs); + assertArtifact(absQawolf); + return channelNames.map((name) => buildChannel(name, absCliJs, absQawolf)); +} + +function assertArtifact(path: string): void { + if (!existsSync(path)) { + throw new Error(`Expected build artifact missing: ${path}`); + } +} + +function buildChannel( + name: ChannelName, + absCliJs: string, + absQawolf: string, +): Channel { + if (name === "node") { + return { label: "node", command: "node", prefixArgs: [absCliJs] }; + } + return { label: "binary", command: absQawolf, prefixArgs: [] }; +} diff --git a/e2e/harness/materialize.ts b/e2e/harness/materialize.ts new file mode 100644 index 000000000..dc5e8903f --- /dev/null +++ b/e2e/harness/materialize.ts @@ -0,0 +1,37 @@ +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, isAbsolute, join, resolve, sep } from "node:path"; + +import type { FlowTemplate, RepoShape, ShapeFile } from "./types.js"; + +// Absolute path to a flow template asset; read as text, never imported. +function flowTemplatePath(flow: FlowTemplate): string { + return join(process.cwd(), "e2e", "fixtures", "flows", `${flow}.flow.ts`); +} + +/** + * Writes a shape's files plus its flow template into a tmp project dir. The flow + * is just another file, written at join(runDir, flowArg) — the exact path passed + * to `flows run`. + */ +export function materialize(shape: RepoShape, projectDir: string): void { + const flowFile: ShapeFile = { + path: join(shape.runDir, shape.flowArg), + content: readFileSync(flowTemplatePath(shape.flow), "utf8"), + }; + for (const file of [...shape.files, flowFile]) { + writeProjectFile(projectDir, file); + } +} + +function writeProjectFile(projectDir: string, file: ShapeFile): void { + if (isAbsolute(file.path)) { + throw new Error(`Shape file path must be relative: ${file.path}`); + } + const root = resolve(projectDir); + const absPath = resolve(root, file.path); + if (absPath !== root && !absPath.startsWith(`${root}${sep}`)) { + throw new Error(`Shape file path escapes project dir: ${file.path}`); + } + mkdirSync(dirname(absPath), { recursive: true }); + writeFileSync(absPath, file.content); +} diff --git a/e2e/harness/report.ts b/e2e/harness/report.ts new file mode 100644 index 000000000..b243de127 --- /dev/null +++ b/e2e/harness/report.ts @@ -0,0 +1,67 @@ +import { intro, log, note, outro } from "@clack/prompts"; + +import type { CaseResult, ChannelName } from "./types.js"; + +export type ReportOptions = { + readonly json: boolean; +}; + +/** + * Renders results and returns a process exit code (0 = all passed, 1 = any + * failed). Clack in a TTY; JSON when `--json` is set or stdout is not a TTY. + */ +export function report( + results: readonly CaseResult[], + options: ReportOptions, +): number { + const exitCode = results.some((result) => !result.passed) ? 1 : 0; + if (options.json || !process.stdout.isTTY) { + console.log(JSON.stringify({ results, exitCode }, undefined, 2)); + return exitCode; + } + renderClack(results, exitCode); + return exitCode; +} + +function renderClack(results: readonly CaseResult[], exitCode: number): void { + intro("e2e: repo-readiness"); + if (results.length === 0) { + note("0 cases — nothing to run (placeholder suite).", "Summary"); + outro("No cases executed."); + return; + } + for (const result of results) { + log.message(formatResultLine(result)); + } + note(formatSummary(results), "Summary"); + outro(exitCode === 0 ? "All cases passed." : "Some cases failed."); +} + +function formatResultLine(result: CaseResult): string { + const mark = result.passed ? "✓" : "✗"; + const seconds = (result.durationMs / 1000).toFixed(1); + const pollution = + result.pollution.length > 0 ? ` pollution:${result.pollution.length}` : ""; + const reasons = result.passed + ? "" + : ` — ${result.assertionFailures.join("; ")}`; + return `${mark} [${result.channel}] ${result.caseName} (${seconds}s${pollution})${reasons}`; +} + +function formatSummary(results: readonly CaseResult[]): string { + const channels: ChannelName[] = ["node", "binary"]; + return channels + .map((channel) => summarizeChannel(channel, results)) + .filter((line): line is string => line !== undefined) + .join("\n"); +} + +function summarizeChannel( + channel: ChannelName, + results: readonly CaseResult[], +): string | undefined { + const forChannel = results.filter((result) => result.channel === channel); + if (forChannel.length === 0) return undefined; + const passed = forChannel.filter((result) => result.passed).length; + return `${channel}: ${passed}/${forChannel.length} passed`; +} diff --git a/e2e/harness/runCase.ts b/e2e/harness/runCase.ts new file mode 100644 index 000000000..dc263f8c0 --- /dev/null +++ b/e2e/harness/runCase.ts @@ -0,0 +1,83 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { + assertCase08Strings, + assertExitAndJunit, + parseJunit, + scanPollution, +} from "./assertions.js"; +import { materialize } from "./materialize.js"; +import { spawnCli } from "./spawnCli.js"; +import { createTmpProject } from "./tmpWorkspace.js"; +import type { CaseResult, Channel, RepoShape } from "./types.js"; + +export type RunCaseOptions = { + readonly noCleanup?: boolean; +}; + +// Generous per-case ceiling — the first case warms the runtime + browser +// download, so this is long enough to never bite a healthy run but bounded so a +// hung CLI can't wedge the matrix forever. +const caseTimeoutMs = 600_000; + +/** + * Runs one shape on one channel in a throwaway project dir that shares the run's + * managed-runtime dir (`runtimeDir`), and collects all assertions into a + * CaseResult. Always cleans up the project unless `noCleanup`, in which case the + * retained path is appended to the output. + */ +export async function runCase( + channel: Channel, + shape: RepoShape, + runtimeDir: string, + options?: RunCaseOptions, +): Promise { + const workspace = createTmpProject(runtimeDir); + try { + materialize(shape, workspace.projectDir); + const runCwd = join(workspace.projectDir, shape.runDir); + const junitPath = join(runCwd, ".junit.xml"); + const startedAt = Date.now(); + const result = await spawnCli( + channel.command, + [ + ...channel.prefixArgs, + "flows", + "run", + shape.flowArg, + "--junit", + junitPath, + ], + { cwd: runCwd, env: workspace.env, timeoutMs: caseTimeoutMs }, + ); + const durationMs = Date.now() - startedAt; + const junit = existsSync(junitPath) + ? parseJunit(readFileSync(junitPath, "utf8")) + : undefined; + const pollution = scanPollution(workspace.projectDir); + const output = `${result.stdout}${result.stderr}`; + const assertionFailures = [ + ...assertExitAndJunit(result.exitCode, junit), + ...(pollution.length > 0 + ? [`project pollution: ${pollution.join(", ")}`] + : []), + ...assertCase08Strings(shape.name, output), + ]; + return { + caseName: shape.name, + channel: channel.label, + passed: assertionFailures.length === 0, + durationMs, + exitCode: result.exitCode, + failures: junit?.failures, + pollution, + assertionFailures, + output: options?.noCleanup + ? `${output}\n[workspace retained: ${workspace.projectDir}]` + : output, + }; + } finally { + if (!options?.noCleanup) workspace.cleanup(); + } +} diff --git a/e2e/harness/spawnCli.ts b/e2e/harness/spawnCli.ts new file mode 100644 index 000000000..b5575fae7 --- /dev/null +++ b/e2e/harness/spawnCli.ts @@ -0,0 +1,50 @@ +import { spawn } from "node:child_process"; + +export type SpawnCliResult = { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; +}; + +export type SpawnCliOptions = { + readonly cwd: string; + readonly env: Record; + // SIGKILL the child after this many ms — guards against a hung CLI or stalled + // browser download wedging the whole matrix with no diagnostic. + readonly timeoutMs?: number; +}; + +/** + * Minimal promise wrapper over node:child_process spawn. Captures exit code and + * decoded stdout/stderr; never rejects — a spawn error (including a timeout + * kill) resolves with exitCode -1 so callers branch on the result instead of + * try/catch. + */ +export function spawnCli( + command: string, + args: readonly string[], + options: SpawnCliOptions, +): Promise { + return new Promise((resolve) => { + const child = spawn(command, [...args], { + cwd: options.cwd, + env: options.env, + timeout: options.timeoutMs, + killSignal: "SIGKILL", + }); + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + child.on("error", (error) => + resolve({ exitCode: -1, stdout, stderr: stderr || error.message }), + ); + child.on("close", (code) => + resolve({ exitCode: code ?? -1, stdout, stderr }), + ); + }); +} diff --git a/e2e/harness/tmpWorkspace.ts b/e2e/harness/tmpWorkspace.ts new file mode 100644 index 000000000..c8de18e10 --- /dev/null +++ b/e2e/harness/tmpWorkspace.ts @@ -0,0 +1,47 @@ +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +export type RuntimeRoot = { + readonly runtimeDir: string; + readonly cleanup: () => void; +}; + +export type TmpProject = { + readonly projectDir: string; + readonly env: Record; + readonly cleanup: () => void; +}; + +/** + * One isolated managed-runtime root for a whole suite run, wired via + * QAWOLF_RUNTIME_DIR so nothing touches the real + * ~/Library/Application Support/qawolf-nodejs. Shared across every case so the + * runtime + browser download warms once per channel (the managed runtime keys + * node vs binary by hash internally). QAWOLF_RUNTIME_DIR points at a subdir so + * its sibling `-runs` staging dir also lands inside the cleaned-up root. + */ +export function createRuntimeRoot(): RuntimeRoot { + const root = mkdtempSync(join(tmpdir(), "qawolf-e2e-rt-")); + const runtimeDir = join(root, "runtime"); + mkdirSync(runtimeDir, { recursive: true }); + return { + runtimeDir, + cleanup: () => rmSync(root, { recursive: true, force: true }), + }; +} + +/** + * A throwaway project dir for one case, sharing the run's managed-runtime dir. + * `cleanup()` removes only this project tree — the runtime root outlives it. + */ +export function createTmpProject(runtimeDir: string): TmpProject { + const tmp = mkdtempSync(join(tmpdir(), "qawolf-e2e-")); + const projectDir = join(tmp, "project"); + mkdirSync(projectDir, { recursive: true }); + return { + projectDir, + env: { ...process.env, QAWOLF_RUNTIME_DIR: runtimeDir }, + cleanup: () => rmSync(tmp, { recursive: true, force: true }), + }; +} diff --git a/e2e/harness/types.ts b/e2e/harness/types.ts new file mode 100644 index 000000000..4c6e25435 --- /dev/null +++ b/e2e/harness/types.ts @@ -0,0 +1,58 @@ +/** The two ways the CLI ships: Node + dist/cli.js, or a compiled standalone binary. */ +export type ChannelName = "node" | "binary"; + +/** + * A built CLI artifact driven by reference (no global install). Spawn + * `command` with `[...prefixArgs, ...cliArgs]`. The node channel runs the real + * `node` binary against `dist/cli.js`; the binary channel runs `dist/qawolf` + * directly with no prefix. + */ +export type Channel = { + readonly label: ChannelName; + readonly command: string; + readonly prefixArgs: readonly string[]; +}; + +/** Which flow template a shape drops into its project tree. */ +export type FlowTemplate = "simpleNav" | "nativeAndVersioned"; + +/** A single file written into a shape's tmp project (package.json, lockfiles, etc.). */ +export type ShapeFile = { readonly path: string; readonly content: string }; + +/** + * A generated project shape exercising one repo layout. `files` excludes the + * flow itself; the flow template named by `flow` is written at + * `join(runDir, flowArg)`. `runDir` is the subdir to run `flows run` from ("" = + * project root); `flowArg` is the flow path relative to that run dir. + */ +export type RepoShape = { + readonly name: string; + readonly proves: string; + readonly files: readonly ShapeFile[]; + readonly flow: FlowTemplate; + readonly runDir: string; + readonly flowArg: string; +}; + +/** A named, data-driven set of cases run across one or more channels. */ +export type Suite = { + readonly name: string; + readonly cases: readonly RepoShape[]; + readonly channels: readonly ChannelName[]; +}; + +/** Outcome of running one shape on one channel. */ +export type CaseResult = { + readonly caseName: string; + readonly channel: ChannelName; + readonly passed: boolean; + readonly durationMs: number; + readonly exitCode: number; + // Parsed from the JUnit `failures="N"` attribute; undefined when no JUnit was produced. + readonly failures: number | undefined; + // Offending node_modules paths found in the project; empty means clean. + readonly pollution: readonly string[]; + // Human-readable reasons the case failed; empty means it passed. + readonly assertionFailures: readonly string[]; + readonly output: string; +}; diff --git a/e2e/run.ts b/e2e/run.ts new file mode 100644 index 000000000..7a3a02a6f --- /dev/null +++ b/e2e/run.ts @@ -0,0 +1,98 @@ +import { resolveChannels } from "./harness/channels.js"; +import { report } from "./harness/report.js"; +import { runCase } from "./harness/runCase.js"; +import { createRuntimeRoot } from "./harness/tmpWorkspace.js"; +import type { CaseResult, ChannelName, Suite } from "./harness/types.js"; +import { allSuites, getSuite } from "./suites/index.js"; + +type CliArgs = { + readonly suiteName: string | undefined; + readonly channel: "node" | "binary" | "both"; + readonly noCleanup: boolean; + readonly json: boolean; +}; + +function parseArgs(argv: readonly string[]): CliArgs { + let suiteName: string | undefined; + let channel: CliArgs["channel"] = "both"; + let noCleanup = false; + let json = false; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--channel") { + const value = argv[++i]; + if (value !== "node" && value !== "binary" && value !== "both") { + throw new Error( + `--channel must be node|binary|both, got: ${value ?? "(missing)"}`, + ); + } + channel = value; + } else if (arg === "--no-cleanup") noCleanup = true; + else if (arg === "--json") json = true; + else if (arg !== undefined && !arg.startsWith("--")) suiteName = arg; + } + return { suiteName, channel, noCleanup, json }; +} + +function resolveSuites(suiteName: string | undefined): Suite[] { + if (suiteName === undefined) return allSuites(); + const suite = getSuite(suiteName); + if (!suite) { + const known = allSuites() + .map((registered) => registered.name) + .join(", "); + throw new Error(`Unknown suite: ${suiteName}. Known suites: ${known}`); + } + return [suite]; +} + +function selectChannelNames( + suite: Suite, + channel: CliArgs["channel"], +): ChannelName[] { + if (channel === "both") return [...suite.channels]; + return suite.channels.includes(channel) ? [channel] : []; +} + +async function runSuite(suite: Suite, args: CliArgs): Promise { + // Skip the build for an empty suite — keeps the placeholder fast and build-free. + if (suite.cases.length === 0) return []; + const channelNames = selectChannelNames(suite, args.channel); + if (channelNames.length === 0) { + throw new Error( + `Suite "${suite.name}" does not support channel "${args.channel}"`, + ); + } + const channels = await resolveChannels(channelNames); + // One shared managed-runtime root warms once per channel (see createRuntimeRoot). + const runtime = createRuntimeRoot(); + const results: CaseResult[] = []; + try { + for (const channel of channels) { + for (const shape of suite.cases) { + results.push( + await runCase(channel, shape, runtime.runtimeDir, { + noCleanup: args.noCleanup, + }), + ); + } + } + } finally { + if (!args.noCleanup) runtime.cleanup(); + } + return results; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + const results: CaseResult[] = []; + for (const suite of resolveSuites(args.suiteName)) { + results.push(...(await runSuite(suite, args))); + } + process.exit(report(results, { json: args.json })); +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/e2e/suites/index.ts b/e2e/suites/index.ts new file mode 100644 index 000000000..79b0a4da5 --- /dev/null +++ b/e2e/suites/index.ts @@ -0,0 +1,16 @@ +import { repoReadinessSuite } from "./repoReadiness.js"; +import type { Suite } from "../harness/types.js"; + +const suites: Record = { + [repoReadinessSuite.name]: repoReadinessSuite, +}; + +/** Looks up a registered suite by name; undefined when unknown. */ +export function getSuite(name: string): Suite | undefined { + return suites[name]; +} + +/** Every registered suite, in registration order. */ +export function allSuites(): Suite[] { + return Object.values(suites); +} diff --git a/e2e/suites/repoReadiness.ts b/e2e/suites/repoReadiness.ts new file mode 100644 index 000000000..c0dabe197 --- /dev/null +++ b/e2e/suites/repoReadiness.ts @@ -0,0 +1,9 @@ +import { repoShapes } from "../fixtures/shapes.js"; +import type { Suite } from "../harness/types.js"; + +/** First suite: every repo shape, on both the node and binary channels. */ +export const repoReadinessSuite: Suite = { + name: "repo-readiness", + cases: repoShapes, + channels: ["node", "binary"], +}; diff --git a/knip.config.ts b/knip.config.ts index 559d1b9f6..8a28463ea 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -8,7 +8,7 @@ const config: KnipConfig = { "src/**/*.fixtures.ts", "src/**/*.testUtils.ts", ], - project: ["src/**/*.ts"], + project: ["src/**/*.ts", "e2e/**/*.ts"], ignoreDependencies: [ // TODO WIZ-10341 follow-up: consumed once the web-flow runner imports it. "@playwright/test", @@ -18,6 +18,8 @@ const config: KnipConfig = { "appium-uiautomator2-driver", ], ignore: [ + // Flow templates aren't imported — they're read as text / executed by a subprocess. + "e2e/fixtures/flows/**", // TODO WIZ-10325: remove once flowsRun consumes more of the runner surface "src/domains/runner/*.ts", // TODO WIZ-10326: remove once Reporter is consumed by more than the console reporter diff --git a/package.json b/package.json index c16b11d1a..291f66f83 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "build": "bun run generate && bun scripts/build.ts", "build:binary": "bun run build && bun scripts/buildBinary.ts", "dev": "bun run src/main.ts", + "e2e": "bun e2e/run.ts", "format": "oxfmt .", "format:check": "oxfmt --check .", "generate": "bun scripts/genDependencyVersions.ts && bun scripts/genSkillMd.ts", diff --git a/tsconfig.json b/tsconfig.json index 11d8387a8..1bd6680b9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -14,6 +14,6 @@ "~/*": ["./src/*"] } }, - "include": ["src", "examples", "scripts"], - "exclude": ["node_modules", "dist"] + "include": ["src", "examples", "scripts", "e2e"], + "exclude": ["node_modules", "dist", "e2e/fixtures/flows"] }