diff --git a/.changeset/fold-downstream-errors.md b/.changeset/fold-downstream-errors.md new file mode 100644 index 00000000..5ed8a7c9 --- /dev/null +++ b/.changeset/fold-downstream-errors.md @@ -0,0 +1,5 @@ +--- +"uptide": patch +--- + +`check` folds a compile error into the breaking site it comes from when the compiler traces its value there (an `errors.map(e => e.message)` under a `result.error.errors` that zod 4 no longer types, and the narrowing that depended on it). The rule's line shows `+N downstream errors`, `--details` lists them under their site, and `--json` carries them as that finding's `downstream`. They are no longer separate breaking sites. `fix` still verifies against every error. diff --git a/.changeset/pack-test-generators.md b/.changeset/pack-test-generators.md new file mode 100644 index 00000000..ad875249 --- /dev/null +++ b/.changeset/pack-test-generators.md @@ -0,0 +1,5 @@ +--- +"uptide": patch +--- + +pack test runs a ground-truth repository's code generators (prisma generate, local graphql-codegen) after its install, as fix does. It fetches the Prisma engines the locked version needs itself, so generators stay off the network. A repository whose generator still fails is incomplete and fails the pack, unless its ground truth declares `withoutGenerated` diff --git a/README.md b/README.md index 19f4b02d..87b0e761 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ See the [pack compatibility matrix](docs/compatibility.md) and the [migration pa | `stripe` | 14+ → latest, API version pinning | 6 | 100% | 92% | 92% | 21 | unkeyed/unkey, nextjs/saas-starter, starl3xx/wallet-to-social, AtlasDevHQ/atlas | verified | | `typescript` | 5 → 6 | 12 | 100% | 99% | 99% | 1399 | m-lyon/recipe, robrechtme/recipes, JosephMaynard/dependency-radar, CroffZ/github-release-notifier, SignalK/signalk-derived-data, shogo82148/typescript-action, Borewit/music-metadata | verified | | `vitest` | 4 → 5 | 6 | 100% | 97% | 97% | 32 | votingworks/vxsuite, vitorvasc/opentelemetry-ecosystem-explorer | verified | -| `zod` | 3 → 4 | 11 | 100% | 93% | 22% | 219 | ugurkocde/AwesomeIntune, howardyang2009/PATH, band-ai/band-sdk-typescript, vzdc-artcc/website, CANShift/canshift-core | verified | +| `zod` | 3 → 4 | 11 | 100% | 87% | 47% | 219 | ugurkocde/AwesomeIntune, howardyang2009/PATH, band-ai/band-sdk-typescript, vzdc-artcc/website, CANShift/canshift-core | verified | 4 more in validation: see the [compatibility matrix](docs/compatibility.md). diff --git a/docs/compatibility.md b/docs/compatibility.md index ebc7021c..77525cc0 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -46,7 +46,7 @@ already run as assisted ([verification gate](packs.md)). | `stripe` | 14+ → latest, API version pinning | 6 | 100% | 92% | 92% | 21 | unkeyed/unkey, nextjs/saas-starter, starl3xx/wallet-to-social, AtlasDevHQ/atlas | verified | | `typescript` | 5 → 6 | 12 | 100% | 99% | 99% | 1399 | m-lyon/recipe, robrechtme/recipes, JosephMaynard/dependency-radar, CroffZ/github-release-notifier, SignalK/signalk-derived-data, shogo82148/typescript-action, Borewit/music-metadata | verified | | `vitest` | 4 → 5 | 6 | 100% | 97% | 97% | 32 | votingworks/vxsuite, vitorvasc/opentelemetry-ecosystem-explorer | verified | -| `zod` | 3 → 4 | 11 | 100% | 93% | 22% | 219 | ugurkocde/AwesomeIntune, howardyang2009/PATH, band-ai/band-sdk-typescript, vzdc-artcc/website, CANShift/canshift-core | verified | +| `zod` | 3 → 4 | 11 | 100% | 87% | 47% | 219 | ugurkocde/AwesomeIntune, howardyang2009/PATH, band-ai/band-sdk-typescript, vzdc-artcc/website, CANShift/canshift-core | verified | | Package | Rewrite rule | Precision | Real true positives | False positives | Note | | --- | --- | ---: | ---: | ---: | --- | @@ -104,11 +104,11 @@ already run as assisted ([verification gate](packs.md)). | `typescript` | `import-assertions` | n/a | 0 | 0 | unexercised: runs assisted | | `typescript` | `module-keyword` | n/a | 0 | 0 | unexercised: runs assisted | | `vitest` | `removed-entrypoints` | n/a | 0 | 0 | unexercised: runs assisted | -| `zod` | `error-issues` | 100% | 52 | 0 | | +| `zod` | `error-issues` | 100% | 51 | 0 | | | `zod` | `error-members` | n/a | 0 | 0 | unexercised: runs assisted | | `zod` | `error-params` | 100% | 15 | 0 | | | `zod` | `record-key` | 100% | 2 | 0 | | -| `zod` | `string-format` | 100% | 19 | 0 | | +| `zod` | `string-format` | 100% | 21 | 0 | | Dependencies without an active pack still receive [generic analysis](concepts.md). diff --git a/docs/concepts.md b/docs/concepts.md index 9f8cc83a..c9059157 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -88,6 +88,32 @@ declaration it traces to. Common causes are grouped into one finding with a site count; `--details` expands sites. For example, TypeScript 7 missing compiler API members form one cause rather than dozens. +A compile error whose value comes from another breaking site is that site's consequence, +not a site of its own. When zod 4 drops `ZodError.errors`, `return { errors: +result.error.errors }` is the breaking site: one edit (`.issues`) fixes it. Every component +that calls that action and maps over the result (`errors.map(e => e.message)`, where `e` is +now implicitly `any`) errors too, and so does the narrowing that depended on it +(`savedAirport` possibly undefined). Those errors fold into the breaking site as +**downstream errors**. The rule: + +- The new error's value is traced with the compiler, never by line distance: from the + expression at the error, through the declaration and initializer of a variable or + destructured binding, the receiver of a callback typed by an `any` value + (`errors.map(e => …)`), the members of a returned object literal, and the call of a + repository function. It folds only when the trace reaches the expression of another new + error that is typed `any` at the target, such as `result.error.errors` (TS2339). +- A repository function's call is followed only when every returned expression whose type + changed between the installed version and the target leads to such an expression. If one + changed return cannot be traced, the trace stops. +- The root must be a breaking finding at that expression (a pack's or the declaration + diff's, or an unattributed compile error). If it is a consequence itself, the error folds + into that one's root. Errors whose trace reaches no reported root stay as they are. +- The default view adds the count to the rule's line (`61 sites, +58 downstream errors`). + `--details` lists each folded error under its root, and `--json` carries them as the + root finding's `downstream`. +- Folding changes what is reported, never what has to compile. `fix` still verifies against + every diagnostic, so a fix that leaves a downstream error broken fails verification. + ## Exit codes For scripts and CI: **0** nothing breaking, **1** breaking changes found, **2** uptide could diff --git a/docs/packs.md b/docs/packs.md index a249f22f..904835ae 100644 --- a/docs/packs.md +++ b/docs/packs.md @@ -253,11 +253,40 @@ They are scored like the others and never count toward `verified`. `uptide pack test` fetches each repository once: a shallow fetch of the pinned commit only, dependencies installed from its lockfile (npm, pnpm, Yarn or bun) with lifecycle scripts -off, and nothing from the repository is ever -executed. They are kept in `~/.cache/uptide/ground-truth` (`UPTIDE_GROUND_TRUTH_CACHE`), so -`pack test --offline` runs without network after the first time. `pnpm packs:fetch` fills +off: no lifecycle script or hook of the repository runs, and the only commands are its code +generators (below). They are kept in `~/.cache/uptide/ground-truth` +(`UPTIDE_GROUND_TRUTH_CACHE`), so `pack test --offline` runs without network after the first +time. `pnpm packs:fetch` fills the cache for every pack (and `--corpus` for the repositories in `fixtures/corpus.json`). +After the install and before `check`, the preparation runs the repository's known code +generators, the ones `fix` runs in its clone (UPT-90): `prisma generate` for a +`schema.prisma`, and GraphQL codegen for a `codegen.ts`/`yml` whose schema is local. On a +user's machine that generated code exists (`prisma generate` has run), so without it `check` +cannot resolve the imports of a generated client and scores those files as `unverified`: the +scoring environment, not the rule, would understate the pack. The generators use the +repository's own binaries, read local files only and reach no network beyond the registry; a +codegen config with a remote schema is not run. Prisma downloads its engine binaries on first +use when its install script did not run (the schema engine, and for Prisma 5 and 6 the query +engine library), so the generators run behind a download mirror nothing listens on. Before +they run, uptide fetches those engines itself, in its own code: the engine commit the locked +`@prisma/engines-version` names (read as data), for this machine's platform, from +binaries.prisma.sh, checked against the published sha256, and kept under the cache's +`prisma-engines/`. Prisma uses them through `PRISMA_SCHEMA_ENGINE_BINARY` and +`PRISMA_QUERY_ENGINE_LIBRARY`, so it generates the same on a fresh runner as on a machine +whose Prisma cache already holds them. The step applies to every pack and every ground-truth +repository alike, never to a single entry. + +A generator that still fails, times out or is not run leaves the repository **incomplete**: +`pack test` does not score it (`incomplete, generated code missing: generator failed: …`), +counts it as "could not be scored", and fails the pack. The one exception is declared in the +ground truth, never assumed: an entry's `withoutGenerated` says why its generators cannot run +at that commit outside the project's own environment (a Prisma config that needs a live +database URL, a codegen config with no schema), and that repository is scored without the +generated code, with the failure and the reason printed. A failed generator runs again on the +next `pack test`, without a new fetch or install. `check` itself still runs no scripts. +Fixtures are synthetic and installed in this tree; they get no generator step. + The cache is large: each entry is a checkout with its dependencies installed. Measured on 2026-10-14, per pack: ai 2.1 GB, lucide-react 3.6, next 1.4, react 4.3, @sentry/browser 0.7, @sentry/nextjs 4.3, @sentry/react 1.4, stripe 2.1, vitest 2.1, zod 2.2. Every pack at once is @@ -299,6 +328,15 @@ An expected site has the severity of the rule or note it is expected under: a ru A missed site counts against breaking recall only when its severity is breaking: a missed `deprecated` note ("needs a look") or a decision is not a missed break. +Downstream errors are not plan sites (see [one root cause, one +finding](concepts.md#one-root-cause-one-finding)). `check` folds a compile error into the +breaking finding whose expression the compiler traces its value to, for example every +`errors.map(e => e.message)` under a `result.error.errors` that zod 4 no longer types. The +root stays a site, and the folded errors are its `downstream`, so `pack test` scores neither +a hit nor a false positive for them. A ground-truth site that `check` folds is a miss. +Ground truth leaves these consequences out, as it always has, and the folding is an engine +rule that applies to every user's `check`, not a scoring exception. + It prints precision and recall for each, `n/a` where there was nothing to measure (nothing predicted, or nothing expected), every false positive as `repo file:line rule (severity)`, every false negative the same way with the severity it was expected at, and sites found under diff --git a/fixtures/deps/issues-v1/index.d.ts b/fixtures/deps/issues-v1/index.d.ts new file mode 100644 index 00000000..69d6974a --- /dev/null +++ b/fixtures/deps/issues-v1/index.d.ts @@ -0,0 +1,12 @@ +/** v1: a parse failure lists its issues as `issues` and as `errors`. */ +export interface Issue { + message: string; +} +export interface Failure { + issues: Issue[]; + errors: Issue[]; +} +export declare function parse( + input: unknown, +): { success: false; error: Failure } | { success: true; data: string }; +export declare function legacy(): number; diff --git a/fixtures/deps/issues-v1/package.json b/fixtures/deps/issues-v1/package.json new file mode 100644 index 00000000..62b49b77 --- /dev/null +++ b/fixtures/deps/issues-v1/package.json @@ -0,0 +1,6 @@ +{ + "name": "issues", + "version": "1.0.0", + "types": "./index.d.ts", + "license": "MIT" +} diff --git a/fixtures/deps/issues-v2/index.d.ts b/fixtures/deps/issues-v2/index.d.ts new file mode 100644 index 00000000..db215e9b --- /dev/null +++ b/fixtures/deps/issues-v2/index.d.ts @@ -0,0 +1,10 @@ +/** v2: `errors` is gone (use `issues`), and so is `legacy`. */ +export interface Issue { + message: string; +} +export interface Failure { + issues: Issue[]; +} +export declare function parse( + input: unknown, +): { success: false; error: Failure } | { success: true; data: string }; diff --git a/fixtures/deps/issues-v2/package.json b/fixtures/deps/issues-v2/package.json new file mode 100644 index 00000000..8196b99c --- /dev/null +++ b/fixtures/deps/issues-v2/package.json @@ -0,0 +1,6 @@ +{ + "name": "issues", + "version": "2.0.0", + "types": "./index.d.ts", + "license": "MIT" +} diff --git a/packages/cli/src/format-check-details.ts b/packages/cli/src/format-check-details.ts index 4a8c96d9..7e239534 100644 --- a/packages/cli/src/format-check-details.ts +++ b/packages/cli/src/format-check-details.ts @@ -74,6 +74,9 @@ function findingLines( if (f.usage.compileError) lines.push(colors.dim(` compiler: ${f.usage.compileError.split('\n')[0]}`)); for (const detail of f.details ?? []) lines.push(colors.dim(` ${detail}`)); + // Errors folded into this site (UPT-99, option cascades): every one, under its root. + for (const d of f.downstream ?? []) + lines.push(colors.dim(` ${d.file}:${d.line} TS${d.code}: ${d.message}`)); const others = (f.sites ?? []).filter((s) => s.line !== f.usage.line); if (others.length > 0) lines.push( diff --git a/packages/cli/src/format-check.ts b/packages/cli/src/format-check.ts index edc1cbdb..4dff93eb 100644 --- a/packages/cli/src/format-check.ts +++ b/packages/cli/src/format-check.ts @@ -288,11 +288,13 @@ function sectionLines(row: Row, colors: Colors): string[] { ) return row.summary || plan.length ? [heading, ...coverage, ...plan] : []; const scope = (g: PlanGroup): string => { + // Errors folded into these sites go with their edits: counted, never sites (UPT-99). + const downstream = g.downstream ? `, +${plural(g.downstream, 'downstream error')}` : ''; if (g.fixes !== g.sites) - return `${plural(g.fixes, 'fix', 'fixes')}, ${plural(g.sites, 'error')}`; + return `${plural(g.fixes, 'fix', 'fixes')}, ${plural(g.sites, 'error')}${downstream}`; const only = g.locations[0]; // A single site is worth naming; the full path is in --details. - return g.sites === 1 && only ? `${basename(only.file)}:${only.line}` : plural(g.sites, 'site'); + return `${g.sites === 1 && only ? `${basename(only.file)}:${only.line}` : plural(g.sites, 'site')}${downstream}`; }; // A package with dozens of distinct changes (a compiler API that was removed) gets its // largest ones here and the rest as a count: --details has every one. diff --git a/packages/cli/src/pack.ts b/packages/cli/src/pack.ts index 58b2c72e..b039f3b5 100644 --- a/packages/cli/src/pack.ts +++ b/packages/cli/src/pack.ts @@ -108,6 +108,7 @@ export function formatPackTest(report: PackTestReport, color = false, limit = In `${head} predicted ${r.predicted.length}, expected ${r.expected.length}, false positives ${r.falsePositives.length}, false negatives ${r.falseNegatives.length}`, ); if (r.installedMismatch) out.push(` ${c.yellow('!')} ${r.installedMismatch}`); + for (const g of r.generatorFailed ?? []) out.push(` ${c.yellow('!')} ${g}`); if (r.companions?.length) out.push( ` with ${r.companions.map((m) => `${m.name} ${m.from} → ${m.to}${m.required ? ` (moved because ${m.reason})` : ''}`).join(', ')}`, diff --git a/packages/core/src/adapters/typescript/cause.test.ts b/packages/core/src/adapters/typescript/cause.test.ts index ef07ae0d..10720e26 100644 --- a/packages/core/src/adapters/typescript/cause.test.ts +++ b/packages/core/src/adapters/typescript/cause.test.ts @@ -313,3 +313,80 @@ describe('a member declared in a repository augmentation the target no longer re }); }); }); + +describe('downstream consequences (UPT-99)', () => { + /** `issues` 1 → 2 drops `Failure.errors` and `legacy`. */ + function issuesConsumer(files: Record): string { + const dir = consumerWith('placeholder.ts', 'export {};\n'); + const tsconfig = JSON.parse(readFileSync(join(dir, 'tsconfig.json'), 'utf8')); + tsconfig.compilerOptions.paths.issues = [join(DEPS, 'issues-v1/index.d.ts')]; + writeFileSync(join(dir, 'tsconfig.json'), JSON.stringify(tsconfig)); + for (const [name, source] of Object.entries(files)) + writeFileSync(join(dir, 'src', name), source); + return dir; + } + + it('traces a value to the removed member it was read from, not an unrelated error beside it', async () => { + const dir = issuesConsumer({ + 'action.ts': [ + "import { parse } from 'issues';", + 'export function save(input: unknown) {', + ' const result = parse(input);', + ' if (!result.success) return { errors: result.error.errors };', + ' return { saved: result.data };', + '}', + '', + ].join('\n'), + 'form.ts': [ + "import { legacy } from 'issues';", + "import { save } from './action';", + 'export function submit(input: unknown): string {', + ' const { saved, errors } = save(input);', + " if (errors) return errors.map((e) => e.message).join('. ');", + ' return saved.toUpperCase();', + '}', + 'export const n = legacy();', + '', + ].join('\n'), + }); + const signal = await compileAgainstTarget({ dir }, 'issues', join(DEPS, 'issues-v2')); + const at = (file: string, code: number) => + signal.diagnostics.find((d) => d.file === `src/${file}` && d.code === code); + const root = at('action.ts', 2339); + expect(root).toMatchObject({ line: 4 }); + expect(root?.origin).toBeUndefined(); + const where = { file: 'src/action.ts', line: 4, column: root?.column }; + // `e` is implicitly `any` because `errors` is: folded. + expect(at('form.ts', 7006)?.origin).toEqual(where); + // `saved` is no longer narrowed by `errors`: the return type changed only at the root. + expect(at('form.ts', 18048)?.origin).toEqual(where); + // The removed `legacy` import is its own break. + expect(at('form.ts', 2305)?.origin).toBeUndefined(); + }); + + it('does not follow a call whose return also changed somewhere it cannot trace', async () => { + const dir = issuesConsumer({ + 'action.ts': [ + "import { parse, legacy } from 'issues';", + 'export function save(input: unknown) {', + ' const result = parse(input);', + ' if (!result.success) return { errors: result.error.errors };', + ' return { saved: result.data, count: legacy() };', + '}', + '', + ].join('\n'), + 'form.ts': [ + "import { save } from './action';", + 'export function submit(input: unknown): string {', + ' const { errors } = save(input);', + " return errors ? errors.map((e) => e.message).join('. ') : '';", + '}', + '', + ].join('\n'), + }); + const signal = await compileAgainstTarget({ dir }, 'issues', join(DEPS, 'issues-v2')); + const implicitAny = signal.diagnostics.find((d) => d.file === 'src/form.ts' && d.code === 7006); + expect(implicitAny).toBeDefined(); + expect(implicitAny?.origin).toBeUndefined(); + }); +}); diff --git a/packages/core/src/adapters/typescript/cause.ts b/packages/core/src/adapters/typescript/cause.ts index 28d66d6c..60169ad9 100644 --- a/packages/core/src/adapters/typescript/cause.ts +++ b/packages/core/src/adapters/typescript/cause.ts @@ -46,12 +46,14 @@ function offendingExpression(tsc: typeof ts, node: ts.Node): ts.Expression | und return n as ts.Expression; } -/** The same declaration in the other program, by file and position. */ +/** The same node in the other program, by file, position and width (`a.b` inside `a.b.c` starts at the same place). */ function counterpart(tsc: typeof ts, decl: ts.Node, program: ts.Program): ts.Node | undefined { const file = program.getSourceFile(decl.getSourceFile().fileName); if (!file) return undefined; + const width = decl.getEnd() - decl.getStart(); let node: ts.Node | undefined = deepestAt(tsc, file, decl.getStart()); - while (node && node.kind !== decl.kind) node = node.parent; + while (node && (node.kind !== decl.kind || node.getEnd() - node.getStart() !== width)) + node = node.parent; return node; } @@ -73,6 +75,51 @@ function typeText(tsc: typeof ts, decl: ts.Node, program: ts.Program): string | } } +/** + * A type as text that does not depend on member order: the installed and target programs may + * list one declaration's members, or a union's, in a different order. Anonymous object types + * are spelled member by member, sorted; named ones by name, as `typeText` prints them. + */ +function shapeText(tsc: typeof ts, node: ts.Node, program: ts.Program): string | undefined { + const checker = program.getTypeChecker(); + const print = (t: ts.Type): string => + checker + .typeToString(t, undefined, tsc.TypeFormatFlags.NoTruncation) + .replace(/import\("[^"]*"\)\./g, '') + .replace(/\b[A-Za-z_$][\w$]*\.(?=[A-Z])/g, ''); + const shape = (t: ts.Type, depth: number): string => { + if (depth > 6) return print(t); + if (t.isUnionOrIntersection()) + return t.types + .map((m) => shape(m, depth + 1)) + .sort() + .join(t.isUnion() ? ' | ' : ' & '); + if (checker.isArrayType(t)) { + const [element] = checker.getTypeArguments(t as ts.TypeReference); + return element ? `${shape(element, depth + 1)}[]` : print(t); + } + const anonymous = !t.aliasSymbol && ['__type', '__object'].includes(t.symbol?.name ?? ''); + if (!(t.flags & tsc.TypeFlags.Object) || !anonymous) return print(t); + if (checker.getSignaturesOfType(t, tsc.SignatureKind.Call).length) return print(t); + return `{ ${checker + .getPropertiesOfType(t) + .map( + (p) => + `${p.name}${p.flags & tsc.SymbolFlags.Optional ? '?' : ''}: ${shape(checker.getTypeOfSymbolAtLocation(p, node), depth + 1)}`, + ) + .sort() + .join('; ')} }`; + }; + try { + return shape( + checker.getTypeAtLocation(tsc.getNameOfDeclaration(node as ts.Declaration) ?? node), + 0, + ); + } catch { + return undefined; + } +} + export function findCause( programs: Programs, diagnostic: ts.Diagnostic, @@ -237,6 +284,181 @@ export function findCause( return undefined; } +/** + * The new diagnostic this one is a consequence of (UPT-99): the value at `diagnostic` is + * followed back with the checker, through variable and destructured declarations, a + * callback's `any` receiver (`errors.map(e => …)`), members of returned object literals and + * calls of repository functions, until it reaches the expression of another new diagnostic + * that is typed `any` at the target (`result.error.errors`, TS2339). A repository function's + * call is followed only when every return whose type changed between the installed version + * and the target leads to such an expression; one it cannot trace and nothing is returned. + * Never by distance: no trace, no origin. + */ +export function flowOrigin( + programs: Programs, + diagnostic: ts.Diagnostic, + file: ts.SourceFile, + repoDir: string, + /** Every new diagnostic, by file name. */ + fresh: ReadonlyMap, +): ts.Diagnostic | undefined { + const tsc = programs.ts; + if (diagnostic.start === undefined) return undefined; + const checker = programs.overlay.getTypeChecker(); + const inRepo = (n: ts.Node): boolean => { + const f = n.getSourceFile().fileName; + return f.startsWith(`${repoDir}/`) && !f.includes('/node_modules/'); + }; + const isAny = (n: ts.Node): boolean => + (checker.getTypeAtLocation(n).flags & tsc.TypeFlags.Any) !== 0; + const declarationOf = (n: ts.Node): ts.Declaration | undefined => { + const symbol = checker.getSymbolAtLocation(n); + const target = + symbol && symbol.flags & tsc.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol; + return target?.declarations?.[0]; + }; + const changed = (n: ts.Node): boolean => { + const before = counterpart(tsc, n, programs.base); + const earlier = before && shapeText(tsc, before, programs.base); + const after = shapeText(tsc, n, programs.overlay); + return earlier === undefined || after === undefined || earlier !== after; + }; + /** `x as SafeParseReturnType<…>` whose import is itself a new error: the cast is `any` because of it. */ + const castRoot = (n: ts.AsExpression): ts.Diagnostic | undefined => { + if (!tsc.isTypeReferenceNode(n.type) || !isAny(n)) return undefined; + let name: ts.EntityName = n.type.typeName; + while (tsc.isQualifiedName(name)) name = name.left; + const decl = checker.getSymbolAtLocation(name)?.declarations?.[0]; + if (!decl || !tsc.isImportSpecifier(decl)) return undefined; + return fresh + .get(decl.getSourceFile().fileName) + ?.find( + (d) => + d !== diagnostic && + d.start !== undefined && + d.start >= decl.getStart() && + d.start < decl.getEnd(), + ); + }; + /** The other new diagnostic at this expression's own position: an access's name, a call's callee. */ + const rootAt = (n: ts.Expression): ts.Diagnostic | undefined => { + const callee = tsc.isCallExpression(n) ? n.expression : n; + const at = tsc.isPropertyAccessExpression(callee) ? callee.name : callee; + const start = at.getStart(); + const root = fresh + .get(n.getSourceFile().fileName) + ?.find((d) => d !== diagnostic && d.start === start); + return root && isAny(n) ? root : undefined; + }; + const seen = new Set(); + /** Every expression here whose type changed leads to an origin: the first one, or none. */ + const explained = ( + expressions: readonly ts.Expression[], + depth: number, + ): ts.Diagnostic | undefined => { + let first: ts.Diagnostic | undefined; + for (const e of expressions) { + if (!changed(e)) continue; + const root = origin(e, depth); + if (!root) return undefined; + first ??= root; + } + return first; + }; + const origin = (start: ts.Node | undefined, depth: number): ts.Diagnostic | undefined => { + let n = start; + while ( + n && + (tsc.isParenthesizedExpression(n) || + tsc.isAwaitExpression(n) || + tsc.isAsExpression(n) || + tsc.isNonNullExpression(n)) + ) { + const root = tsc.isAsExpression(n) ? castRoot(n) : undefined; + if (root) return root; + n = n.expression; + } + if (!n || depth > MAX_HOPS || seen.has(n)) return undefined; + seen.add(n); + if (tsc.isExpression(n)) { + const root = rootAt(n); + if (root) return root; + } + if (tsc.isCallExpression(n)) { + const callee = n.expression; + const decl = declarationOf(tsc.isPropertyAccessExpression(callee) ? callee.name : callee); + if (decl && inRepo(decl)) return explained(returnedExpressions(tsc, decl), depth + 1); + return undefined; + } + if (tsc.isObjectLiteralExpression(n)) { + const members = n.properties.flatMap((p) => { + if (tsc.isPropertyAssignment(p)) return [p.initializer]; + if (tsc.isShorthandPropertyAssignment(p)) return [p.name]; + return []; + }); + return explained(members, depth + 1); + } + if (tsc.isPropertyAccessExpression(n)) { + if (isAny(n.expression)) return origin(n.expression, depth + 1); + // A member of the repository's own object (a returned literal's `errors`): its value. + const symbol = checker.getSymbolAtLocation(n.name); + for (const decl of symbol?.declarations ?? []) { + if (!inRepo(decl) || !tsc.isPropertyAssignment(decl)) continue; + const root = origin(decl.initializer, depth + 1); + if (root) return root; + } + return undefined; + } + if (!tsc.isIdentifier(n)) return undefined; + const value = tsc.isShorthandPropertyAssignment(n.parent) + ? checker.getShorthandAssignmentValueSymbol(n.parent)?.declarations?.[0] + : declarationOf(n); + if (!value || !inRepo(value)) return undefined; + if (tsc.isVariableDeclaration(value)) return origin(value.initializer, depth + 1); + if (tsc.isBindingElement(value)) { + // Destructured: from the value the pattern takes apart. + let holder: ts.Node = value.parent.parent; + while (tsc.isBindingElement(holder)) holder = holder.parent.parent; + return tsc.isVariableDeclaration(holder) ? origin(holder.initializer, depth + 1) : undefined; + } + if (tsc.isParameter(value)) { + // A callback's parameter typed by nothing but an `any` receiver: `errors.map(e => …)`. + const fn = value.parent; + const call = fn.parent; + if ( + (tsc.isArrowFunction(fn) || tsc.isFunctionExpression(fn)) && + tsc.isCallExpression(call) && + call.arguments.some((a) => a === fn) && + tsc.isPropertyAccessExpression(call.expression) && + isAny(call.expression.expression) + ) + return origin(call.expression.expression, depth + 1); + } + return undefined; + }; + return origin(offendingExpression(tsc, deepestAt(tsc, file, diagnostic.start)), 0); +} + +/** Every value a function returns: its expression body, or each `return` outside nested functions. */ +function returnedExpressions(tsc: typeof ts, decl: ts.Declaration): ts.Expression[] { + let body: ts.Node | undefined; + if (tsc.isVariableDeclaration(decl) && decl.initializer) body = decl.initializer; + else if (tsc.isFunctionLike(decl) && 'body' in decl) body = (decl as { body?: ts.Node }).body; + else return []; + if (body && (tsc.isArrowFunction(body) || tsc.isFunctionExpression(body))) { + if (!tsc.isBlock(body.body)) return [body.body]; + body = body.body; + } + if (!body || !tsc.isBlock(body)) return []; + const returned: ts.Expression[] = []; + const visit = (n: ts.Node): void => { + if (tsc.isReturnStatement(n) && n.expression) returned.push(n.expression); + else if (!tsc.isFunctionLike(n)) tsc.forEachChild(n, visit); + }; + tsc.forEachChild(body, visit); + return returned; +} + function findImportDeclaration(tsc: typeof ts, node: ts.Node): ts.ImportDeclaration | undefined { let n: ts.Node | undefined = node; while (n && !tsc.isImportDeclaration(n)) n = n.parent; diff --git a/packages/core/src/adapters/typescript/compile.ts b/packages/core/src/adapters/typescript/compile.ts index 309682e3..ff2a658d 100644 --- a/packages/core/src/adapters/typescript/compile.ts +++ b/packages/core/src/adapters/typescript/compile.ts @@ -21,7 +21,7 @@ import type { import { satisfies } from '../../fetch/range.js'; import { missingGeneratorStep } from '../../fix/generate.js'; import { onReset } from '../../shared-state.js'; -import { augmentationCause, findCause, parameterCause } from './cause.js'; +import { augmentationCause, findCause, flowOrigin, parameterCause } from './cause.js'; import { type Compiler, consumerCompilerDir, @@ -1129,6 +1129,9 @@ async function compileProject( } const shared = new Map(); for (const key of parameterOf.values()) shared.set(key, (shared.get(key) ?? 0) + 1); + const freshByFile = new Map(); + for (const { d, overlaid } of fresh) + freshByFile.set(overlaid.fileName, [...(freshByFile.get(overlaid.fileName) ?? []), d]); const diagnostics: CompileDiagnostic[] = fresh.map(({ d, overlaid }) => { const diagnostic = toDiagnostic(d, overlaid, repo.dir); const parameter = parameterOf.get(d); @@ -1154,6 +1157,15 @@ async function compileProject( const root = augmentation ?? (parameter !== undefined ? byParameter.get(parameter) : undefined); if (root) diagnostic.root = root; + // A value the compiler traces to another new error's expression is that error's + // consequence (UPT-99); an anchor that is the one edit (an option, a parameter) keeps it. + if (!cause?.config && !cause?.anchorOnly) { + const from = flowOrigin(programs, d, overlaid, repo.dir, freshByFile); + if (from?.file) { + const at = toDiagnostic(from, from.file, repo.dir); + diagnostic.origin = { file: at.file, line: at.line, column: at.column }; + } + } return diagnostic; }); return { diff --git a/packages/core/src/check/check.ts b/packages/core/src/check/check.ts index 24b405f3..0c4a75bb 100644 --- a/packages/core/src/check/check.ts +++ b/packages/core/src/check/check.ts @@ -66,7 +66,7 @@ import { foldOptionCascades } from './option-cascades.js'; import { mapWithLimit, mapWithSerialRetry } from './pool.js'; import { isBehind, rankCandidates } from './rank.js'; import { ownTypesFrom } from './removed.js'; -import { foldSharedRoots, groupRootCauses } from './root-cause.js'; +import { foldConsequences, foldSharedRoots, groupRootCauses } from './root-cause.js'; import { runtimeChangeFindings } from './runtime-changes.js'; import { type StandIns, standInLeads } from './stand-ins.js'; import { confirmBreaking, evidenceOf, tierOf } from './tier.js'; @@ -474,6 +474,17 @@ export async function check(opts: CheckOptions): Promise { findings: report.findings, }); } + // Compile errors the compiler traces to another breaking site go with that site's edit (UPT-99). + for (const report of results.flat()) { + const findings = foldConsequences( + report.findings, + report.workspace === '.' || report.workspace === '*' ? '' : `${report.workspace}/`, + ); + const folded = findings.length < report.findings.length; + report.findings = findings; + // The verdict counts breaking findings: recount without the folded ones. + if (folded && report.verdict) report.verdict = verdictOf(report, report.verdict.notVerified); + } const order = new Map(workspaces.map((w, i) => [w, i])); const packages = mergeAcrossWorkspaces( results.flat().sort((a, b) => (order.get(a.workspace) ?? 0) - (order.get(b.workspace) ?? 0)), diff --git a/packages/core/src/check/root-cause.test.ts b/packages/core/src/check/root-cause.test.ts index 7e30edfd..1e46c0c0 100644 --- a/packages/core/src/check/root-cause.test.ts +++ b/packages/core/src/check/root-cause.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import type { Finding, PackageReport } from '../domain/report.js'; import { planPackage } from '../fix/plan.js'; import { sitesOf } from './check.js'; -import { callSitesLine, foldSharedRoots, groupRootCauses } from './root-cause.js'; +import { callSitesLine, foldConsequences, foldSharedRoots, groupRootCauses } from './root-cause.js'; it('groups missing TypeScript API members without losing sites or unrelated diagnostics', () => { const finding = (line: number, code = 2339): Finding => ({ @@ -199,3 +199,112 @@ describe('foldSharedRoots', () => { ).toBe('2 call sites'); }); }); + +describe('foldConsequences (UPT-99)', () => { + const at = ( + file: string, + line: number, + column: number, + code: number, + extra: Partial = {}, + ): Finding => ({ + change: { + package: 'issues', + from: '1.0.0', + to: '2.0.0', + path: `TS${code}`, + kind: 'type', + severity: 'breaking', + source: 'types', + confidence: 1, + }, + usage: { + file, + line, + column, + endLine: line, + endColumn: column + 6, + symbolPath: `TS${code}`, + access: 'read', + via: 'inferred', + snippet: '', + compileCode: code, + compileError: `error ${code}`, + }, + severity: 'breaking', + confidence: 1, + fixability: 'unknown', + reason: 'compile error not attributed to a known API change', + ...extra, + }); + const root = at('src/action.ts', 4, 52, 2339, { + change: { + package: 'issues', + from: '1.0.0', + to: '2.0.0', + path: 'Failure#errors', + kind: 'removed', + severity: 'breaking', + source: 'types', + confidence: 1, + }, + }); + const origin = { file: 'src/action.ts', line: 4, column: 52 }; + + it('folds traced errors into the root, keeps the rest, and leaves no origin behind', () => { + const implicitAny = at('src/form.ts', 5, 31, 7006, { origin }); + const narrowing = at('src/form.ts', 6, 10, 18048, { origin }); + // Traced through the narrowing error, which is a consequence itself: same root. + const chained = at('src/form.ts', 9, 3, 2532, { + origin: { file: 'src/form.ts', line: 6, column: 10 }, + }); + const unrelated = at('src/form.ts', 1, 10, 2305); + const orphan = at('src/form.ts', 8, 3, 7006, { + origin: { file: 'src/other.ts', line: 1, column: 1 }, + }); + const out = foldConsequences( + [root, implicitAny, narrowing, chained, unrelated, orphan], + 'apps/web/', + ); + expect(out).toHaveLength(3); + const [folded, ...rest] = out; + expect(folded?.downstream?.map((d) => [d.file, d.line, d.code])).toEqual([ + ['apps/web/src/form.ts', 5, 7006], + ['apps/web/src/form.ts', 6, 18048], + ['apps/web/src/form.ts', 9, 2532], + ]); + expect(folded?.details?.at(-1)).toMatch(/^\+3 downstream errors/); + // Nothing breaking at its origin: it stays, without the internal field. + const { origin: _, ...orphanAsIs } = orphan; + expect(rest).toEqual([unrelated, orphanAsIs]); + expect(out.some((f) => 'origin' in f)).toBe(false); + // A consequence is never a site: the plan counts it under its root, not as one. + const report: PackageReport = { + name: 'issues', + installed: '1.0.0', + target: '2.0.0', + latest: '2.0.0', + workspace: '.', + majorsBehind: 1, + findings: [folded as Finding], + status: 'breaking', + callSitesChecked: 1, + unanalyzed: [], + notes: [], + timing: { fetchMs: 0, diffMs: 0, usagesMs: 0, compileMs: 0 }, + }; + const [group] = planPackage(report); + expect(group).toMatchObject({ sites: 1, downstream: 3 }); + expect(sitesOf(folded as Finding)).toBe(1); + }); + + it('does not fold into a root that is not breaking, or a loop', () => { + const deprecated = { ...root, severity: 'deprecated' as const }; + const a = at('src/a.ts', 1, 1, 7006, { origin: { file: 'src/b.ts', line: 1, column: 1 } }); + const b = at('src/b.ts', 1, 1, 7006, { origin: { file: 'src/a.ts', line: 1, column: 1 } }); + const c = at('src/form.ts', 5, 31, 7006, { origin }); + const out = foldConsequences([deprecated, a, b, c]); + expect(out).toHaveLength(4); + expect(out.every((f) => f.downstream === undefined)).toBe(true); + }); +}); diff --git a/packages/core/src/check/root-cause.ts b/packages/core/src/check/root-cause.ts index db4a867e..32783630 100644 --- a/packages/core/src/check/root-cause.ts +++ b/packages/core/src/check/root-cause.ts @@ -1,9 +1,81 @@ +import { posix } from 'node:path'; import type { DownstreamSite, Finding, PackageReport, SharedRoot } from '../domain/report.js'; +import type { SitePosition } from '../domain/usage.js'; import { describeCluster } from './unattributed.js'; import { parseVersion } from './version.js'; const rootKey = (r: SharedRoot): string => `${r.file}:${r.line}:${r.name}`; +/** + * A compile error the compiler traces to another new error's expression (`origin`, set by + * `flowOrigin`, UPT-99) is a consequence of that site: it folds into the breaking finding + * there as one of its `downstream` errors and is no longer a site. The root is the innermost + * breaking finding (not a cause cluster) whose span holds the origin. A consequence of a + * consequence goes to the first one's root. With no breaking finding at the origin, or a + * trace that loops, the error stays as it was. `prefix` makes workspace-relative files + * repository-relative, as `downstream` is. + */ +export function foldConsequences(findings: Finding[], prefix = ''): Finding[] { + if (!findings.some((f) => f.origin)) return findings; + const before = (aLine: number, aCol: number, bLine: number, bCol: number): boolean => + aLine < bLine || (aLine === bLine && aCol <= bCol); + const holds = (f: Finding, at: SitePosition): boolean => + f.usage.file === at.file && + before(f.usage.line, f.usage.column, at.line, at.column) && + before(at.line, at.column, f.usage.endLine, f.usage.endColumn); + const span = (f: Finding): number => + (f.usage.endLine - f.usage.line) * 1e6 + f.usage.endColumn - f.usage.column; + const rootAt = (at: SitePosition): Finding | undefined => + findings + .filter((r) => r.severity === 'breaking' && r.change.kind !== 'cause' && holds(r, at)) + .sort((a, b) => span(a) - span(b))[0]; + const rootOf = (f: Finding): Finding | undefined => { + const seen = new Set([f]); + let current = f; + while (current.origin) { + const next = rootAt(current.origin); + if (!next) break; + if (seen.has(next)) return undefined; + seen.add(next); + current = next; + } + return current === f ? undefined : current; + }; + const into = new Map(); + for (const f of findings) { + const root = f.origin && rootOf(f); + if (root) into.set(root, [...(into.get(root) ?? []), f]); + } + const folded = new Set([...into.values()].flat()); + return findings.flatMap((f) => { + if (folded.has(f)) return []; + const { origin: _, ...rest } = f; + const under = into.get(f); + if (!under) return [rest]; + const downstream: DownstreamSite[] = under.map((c) => ({ + file: posix.normalize(`${prefix}${c.usage.file}`), + line: c.usage.line, + column: c.usage.column, + snippet: c.usage.snippet, + code: c.usage.compileCode ?? Number(/^TS(\d+)$/.exec(c.change.path)?.[1] ?? 0), + message: (c.usage.compileError ?? c.reason).split('\n')[0] ?? c.reason, + })); + const n = downstream.length; + return [ + { + ...rest, + downstream: [...(f.downstream ?? []), ...downstream].sort( + (a, b) => a.file.localeCompare(b.file) || a.line - b.line, + ), + details: [ + ...(f.details ?? []), + `+${n} downstream error${n === 1 ? '' : 's'}: the compiler traces ${n === 1 ? 'its value' : 'their values'} to this site, and the edit here resolves ${n === 1 ? 'it' : 'them'}`, + ], + }, + ]; + }); +} + /** `3 call sites in 2 workspaces`: the line under a shared root's reason. */ export function callSitesLine(sites: readonly DownstreamSite[]): string { const workspaces = new Set(sites.map((s) => s.workspace).filter((w) => w !== undefined)); diff --git a/packages/core/src/check/unattributed.ts b/packages/core/src/check/unattributed.ts index 4308f96b..4614d76b 100644 --- a/packages/core/src/check/unattributed.ts +++ b/packages/core/src/check/unattributed.ts @@ -46,7 +46,8 @@ export function unattributedFindings( const clusters = new Map(); const single: CompileDiagnostic[] = []; for (const d of diagnostics) { - if (!d.cause) { + // A consequence of another new error folds into that error's finding (root-cause.ts). + if (!d.cause || d.origin) { single.push(d); continue; } @@ -153,6 +154,7 @@ function plainFindings( change, usage, ...(d.root ? { root: sharedRoot(d.root) } : {}), + ...(d.origin ? { origin: d.origin } : {}), severity: inconclusive ? 'unverified' : 'breaking', confidence: 1, fixability: 'unknown', diff --git a/packages/core/src/domain/report.ts b/packages/core/src/domain/report.ts index 9f036a34..2d780b8c 100644 --- a/packages/core/src/domain/report.ts +++ b/packages/core/src/domain/report.ts @@ -1,7 +1,13 @@ import type { ErrorCode } from '../errors.js'; import type { Change, Severity } from './change.js'; import type { RuntimeChange, RuntimeLoad } from './runtime.js'; -import type { CompileCoverage, CompileDiagnostic, Unanalyzed, Usage } from './usage.js'; +import type { + CompileCoverage, + CompileDiagnostic, + SitePosition, + Unanalyzed, + Usage, +} from './usage.js'; /** `none`: an additive finding, nothing to change; it counts in callSitesChecked only. */ export type Fixability = 'mechanical' | 'assisted' | 'manual' | 'unknown' | 'none'; @@ -49,6 +55,11 @@ export interface Finding { * declaration (`anchorOnly`, the sites as `downstream`), which carries `root` too. */ root?: SharedRoot; + /** + * Internal: the new diagnostic this compile error is a consequence of (UPT-99). Folded into + * the breaking finding there as one of its `downstream` errors, then removed. + */ + origin?: SitePosition; /** * For a `module-format` finding: every `require()` site of the package in this file, the * finding's own first. Switching a file to `import()` is one unit of work however many @@ -106,6 +117,8 @@ export interface PlanGroup { sites: number; /** Edits expected: fewer than `sites` when one edit resolves several errors. */ fixes: number; + /** Compile errors folded into these sites as their consequences (UPT-99): not sites, not edits. */ + downstream?: number; /** Repository-relative, in file and line order. */ locations: { file: string; line: number }[]; /** For deprecations: the deprecated names as written (`.email`) and their call counts. */ diff --git a/packages/core/src/domain/usage.ts b/packages/core/src/domain/usage.ts index aa913f50..533a20cd 100644 --- a/packages/core/src/domain/usage.ts +++ b/packages/core/src/domain/usage.ts @@ -115,6 +115,19 @@ export interface CompileDiagnostic { * workspaces that trace to the same declaration fold into one finding at it (check/root-cause.ts). */ root?: DiagnosticCause; + /** + * The other new diagnostic whose expression the compiler traces this one's value to + * (`flowOrigin`, UPT-99), workspace-relative like `file`. `check` folds this one into the + * breaking finding at that place as a downstream error (check/root-cause.ts). + */ + origin?: SitePosition; +} + +/** A position in a file, 1-based. */ +export interface SitePosition { + file: string; + line: number; + column: number; } export interface DiagnosticCause { diff --git a/packages/core/src/fix/generate.ts b/packages/core/src/fix/generate.ts index 8dfb850b..fd64bc2e 100644 --- a/packages/core/src/fix/generate.ts +++ b/packages/core/src/fix/generate.ts @@ -30,7 +30,12 @@ interface Generator { * local files only: `prisma generate` contacts no database, and a codegen config with a remote * schema is reported, not run. It runs in fix's clone, with the repository's own binary. */ -export async function generateClients(root: string, workspaces: string[]): Promise { +export async function generateClients( + root: string, + workspaces: string[], + /** Added to each generator's environment: ground truth blocks downloads with it. */ + env: Record = {}, +): Promise { const out: Generated[] = []; for (const workspace of workspaces) { const dir = join(root, workspace); @@ -42,7 +47,7 @@ export async function generateClients(root: string, workspaces: string[]): Promi } const bin = findBin(dir, root, g.bin); if (!bin) continue; - const result = await command(dir, bin, g.args, 180_000, g.env); + const result = await command(dir, bin, g.args, 180_000, { ...g.env, ...env }); out.push({ workspace, command: typed, diff --git a/packages/core/src/fix/plan.ts b/packages/core/src/fix/plan.ts index 1b159555..660b3cf0 100644 --- a/packages/core/src/fix/plan.ts +++ b/packages/core/src/fix/plan.ts @@ -71,6 +71,11 @@ export function planPackage( const severity = first.finding.severity as PlanGroup['severity']; const count = (o: Outcome): number => sites.filter((s) => s.outcome === o).length; const assisted = sites.filter((s) => s.outcome !== 'mechanical'); + // A cluster's sites are its downstream errors; anywhere else they are consequences. + const downstream = sites.reduce( + (n, s) => n + (s.finding.change.kind === 'cause' ? 0 : (s.finding.downstream?.length ?? 0)), + 0, + ); const group: PlanGroup = { rule: first.rule, title: titleOf(first.rule, sites, usable ? summaryOf(pack, first.rule) : undefined), @@ -83,6 +88,7 @@ export function planPackage( locations: sites .map((s) => ({ file: s.finding.usage.file, line: s.finding.usage.line })) .sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line), + ...(downstream > 0 ? { downstream } : {}), detail: (first.finding.usage.compileError ?? first.finding.reason).split('\n')[0] ?? '', }; if (severity === 'deprecated') { diff --git a/packages/core/src/fix/select.ts b/packages/core/src/fix/select.ts index 8a4a19ae..10950a6d 100644 --- a/packages/core/src/fix/select.ts +++ b/packages/core/src/fix/select.ts @@ -23,9 +23,10 @@ export function selectedFindings( continue; // A cluster's sites are its downstream errors, unless the anchor is the one site to edit. if (finding.change.kind === 'cause' && !finding.anchorOnly) { - for (const d of finding.downstream ?? []) + const { downstream, ...cluster } = finding; + for (const d of downstream ?? []) selected.push({ - ...finding, + ...cluster, change: { ...finding.change, kind: 'type', path: `TS${d.code}` }, usage: { ...finding.usage, diff --git a/packages/core/src/packs/contract.ts b/packages/core/src/packs/contract.ts index 7d92dfa0..6b28c1d3 100644 --- a/packages/core/src/packs/contract.ts +++ b/packages/core/src/packs/contract.ts @@ -140,6 +140,13 @@ export interface GroundTruthRepo { * the package upgraded alone is an install the real upgrade never had. */ with?: Record; + /** + * Why this repository is scored without its generated code: a generator that cannot run at + * this commit outside its own environment (a Prisma config that needs a live database URL, a + * codegen config with no schema). Without it, a generator that fails leaves the repository + * unscored and fails `pack test` (UPT-95). + */ + withoutGenerated?: string; why: string; /** Repository-relative `file`, 1-based `line`, and the rule (or behavior note) id. */ findings: GroundTruthFinding[]; diff --git a/packages/core/src/packs/ioredis/ground-truth.json b/packages/core/src/packs/ioredis/ground-truth.json index 353d7ca8..c776bf3b 100644 --- a/packages/core/src/packs/ioredis/ground-truth.json +++ b/packages/core/src/packs/ioredis/ground-truth.json @@ -82,6 +82,7 @@ "migration": "https://github.com/simondietschi/dokunc/commit/c47539089296efdb02a3093bb650185b4861f297", "from": "5.11.1", "to": "6.0.0", + "withoutGenerated": "its packages/db/prisma.config.ts reads DATABASE_URL, which only the project's own environment sets, so prisma generate stops while loading the config", "why": "merged in PR #15 (03ae5da): bump bb72cd7 extends the credential-redaction test to HELLO 3 AUTH; c475390 restores the v5 retryStrategy at the collab client after measuring 10-15 s command stalls; @hocuspocus/extension-redis keeps its own ioredis 5.6.1", "findings": [ { @@ -125,6 +126,7 @@ "migration": "https://github.com/kazumadev619-dev/FishingConditionsApp/commit/cdecc03c3c04d0e4f08351ceef6b10df6f1bfd81", "from": "5.11.1", "to": "6.0.0", + "withoutGenerated": "its prisma.config.ts reads DATABASE_URL_DIRECT, which only the project's own environment sets, so prisma generate stops while loading the config", "why": "npm upgrade merged in PR #235 (ebf69b5) that removes the stale @types/ioredis stub and keeps its own retryStrategy: no source change, no compiler diagnostic", "findings": [] } diff --git a/packages/core/src/packs/ioredis/verification.json b/packages/core/src/packs/ioredis/verification.json index 0541c791..25e0b93c 100644 --- a/packages/core/src/packs/ioredis/verification.json +++ b/packages/core/src/packs/ioredis/verification.json @@ -1,6 +1,6 @@ { "status": "candidate", - "truth": "sha256-ce9e34cf184316cce69972cf34de3a269d38fc37d077eb8936586e55bb7328de", + "truth": "sha256-0951cfb69d5bb36f19f015334cca731c01a3b63e2e93cd65dbacf37b95fe31f9", "repos": 5, "breaking": { "precision": 1, diff --git a/packages/core/src/packs/react-router/ground-truth.json b/packages/core/src/packs/react-router/ground-truth.json index 2bc5b302..71f30de0 100644 --- a/packages/core/src/packs/react-router/ground-truth.json +++ b/packages/core/src/packs/react-router/ground-truth.json @@ -170,6 +170,7 @@ "migration": "https://github.com/getlago/lago-front/commit/6055a5a3c8076522235fe2da452a209e99af8541", "from": "6.30.6", "to": "7.18.3", + "withoutGenerated": "graphql-codegen fails at this commit: the local schema it reads has no Query root type", "with": { "react-router-dom": "7.18.3" }, diff --git a/packages/core/src/packs/react-router/verification.json b/packages/core/src/packs/react-router/verification.json index 5417e484..d7fee267 100644 --- a/packages/core/src/packs/react-router/verification.json +++ b/packages/core/src/packs/react-router/verification.json @@ -1,6 +1,6 @@ { "status": "verified", - "truth": "sha256-2b693b9c3d4700152a748eaa3605836c48251cffd77ef3b9e3c2435abcf3d3c4", + "truth": "sha256-3d0310dc1788350769a25b827f81ff09e98017642473b832a716a352914b5f7c", "repos": 2, "breaking": { "precision": 1, diff --git a/packages/core/src/packs/tooling/pack-test.test.ts b/packages/core/src/packs/tooling/pack-test.test.ts index baff4380..58ebf956 100644 --- a/packages/core/src/packs/tooling/pack-test.test.ts +++ b/packages/core/src/packs/tooling/pack-test.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -17,6 +17,7 @@ import { type RepoScore, testPack, } from './pack-test.js'; +import { cachedRepoDir } from './truth.js'; const root = fileURLToPath(new URL('../../../../../', import.meta.url)); const verification = JSON.parse( @@ -364,3 +365,63 @@ it('writes a record as the formatter would: a short list inline, a long one a li expect(`${formatRecord(entry.verification)}\n`, entry.dir).toBe(readFileSync(file, 'utf8')); } }); + +it('fails a repository whose generated code is missing, unless the ground truth says why (UPT-95)', async () => { + const tmp = mkdtempSync(join(tmpdir(), 'uptide-generated-')); + const cacheDir = join(tmp, 'cache'); + const repo = { repo: 'example/prisma-app', commit: 'c'.repeat(40) }; + const truth = (withoutGenerated?: string) => { + mkdirSync(join(tmp, 'packages/core/src/packs/demo'), { recursive: true }); + writeFileSync( + join(tmp, 'packages/core/src/packs/demo/ground-truth.json'), + JSON.stringify({ + package: 'demo', + repos: [ + { ...repo, from: '1.0.0', to: '2.0.0', why: 'synthetic', findings: [], withoutGenerated }, + ], + }), + ); + }; + // Installed and cached; its Prisma generator fails the way a missing engine does. + const dir = cachedRepoDir(repo, cacheDir); + mkdirSync(join(dir, 'prisma'), { recursive: true }); + mkdirSync(join(dir, 'node_modules/.bin'), { recursive: true }); + writeFileSync(join(dir, 'package.json'), '{"name":"app"}'); + writeFileSync(join(dir, 'prisma/schema.prisma'), 'generator client { provider = "x" }'); + const bin = join(dir, 'node_modules/.bin/prisma'); + writeFileSync( + bin, + '#!/bin/sh\necho "Error: request to http://127.0.0.1:9/ failed" >&2\nexit 1\n', + ); + chmodSync(bin, 0o755); + writeFileSync( + join(dir, '.uptide-ground-truth.json'), + JSON.stringify({ ...repo, installed: ['.'] }), + ); + const pack = { + ...aiPack, + name: 'demo', + rules: [], + behavior: [], + companions: undefined, + releaseGroup: undefined, + meta: { ...aiPack.meta, minBreakingSites: undefined }, + }; + const run = () => + testPack({ dir: 'demo', pack, verification }, { root: tmp, cacheDir, offline: true }); + + truth(); + const incomplete = await run(); + expect(incomplete.repos[0]?.error).toMatch( + /^incomplete, generated code missing: generator failed: prisma generate/, + ); + expect(incomplete.gate.reasons).toContain('example/prisma-app@cccccccccccc could not be scored'); + expect(incomplete.passed).toBe(false); + + truth('its prisma.config.ts needs a live DATABASE_URL'); + const declared = await run(); + expect(declared.repos[0]?.error ?? '').not.toMatch(/^incomplete/); + expect(declared.repos[0]?.generatorFailed?.[0]).toMatch( + /; scored without it: its prisma.config.ts needs a live DATABASE_URL$/, + ); +}); diff --git a/packages/core/src/packs/tooling/pack-test.ts b/packages/core/src/packs/tooling/pack-test.ts index 72e23e77..6363b982 100644 --- a/packages/core/src/packs/tooling/pack-test.ts +++ b/packages/core/src/packs/tooling/pack-test.ts @@ -24,6 +24,7 @@ import { type EnsureOptions, ensureFixture, ensureRepo, + generatorFailures, label, pinnedRepo, truthProblems, @@ -67,6 +68,11 @@ export interface RepoScore { error?: string; /** What the lockfile says is installed, when it is not the ground truth's `from`. */ installedMismatch?: string; + /** + * Code generators that did not run or failed. Scored without their code only when the ground + * truth declares it (`withoutGenerated`); otherwise `error` says the repository is incomplete. + */ + generatorFailed?: string[]; /** What `check` moves with the package there, at versions that agree (`fix` installs them together). */ companions?: Companion[]; /** Packages of the scope `check` leaves where they are (`check/companions.ts`). */ @@ -351,6 +357,15 @@ async function scoreRepo( timing.fetchMs = since(start); start = performance.now(); dir = await ensureRepo(pinned, options); + const failed = generatorFailures(pinned, options.cacheDir); + if (failed.length) + base.generatorFailed = entry.withoutGenerated + ? failed.map((f) => `${f}; scored without it: ${entry.withoutGenerated}`) + : failed; + // Scored without the generated code only where the ground truth says why: otherwise the + // repository is incomplete, and never scored as if it were not. + if (failed.length && !entry.withoutGenerated) + throw new Error(`incomplete, generated code missing: ${failed.join('; ')}`); } timing.installMs = since(start); const project = entry.directory ? join(dir, entry.directory) : dir; diff --git a/packages/core/src/packs/tooling/prisma-engines.test.ts b/packages/core/src/packs/tooling/prisma-engines.test.ts new file mode 100644 index 00000000..4574f1ec --- /dev/null +++ b/packages/core/src/packs/tooling/prisma-engines.test.ts @@ -0,0 +1,117 @@ +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { afterEach, describe, expect, it } from 'vitest'; +import { binaryTarget, prismaEngineEnv } from './prisma-engines.js'; + +const HASH = '0edf323efd1d98336f3f0a68684b56f689b900d3'; +const TARGET = 'debian-openssl-3.0.x'; + +/** A project with Prisma `version` installed, its engine commit where Prisma's own packages keep it. */ +function project(version: string): string { + const dir = mkdtempSync(join(tmpdir(), 'uptide-engines-')); + const pkg = (name: string, extra: object = {}) => { + mkdirSync(join(dir, 'node_modules', name), { recursive: true }); + writeFileSync( + join(dir, 'node_modules', name, 'package.json'), + JSON.stringify({ name, version, ...extra }), + ); + }; + pkg('prisma'); + pkg('@prisma/engines'); + pkg('@prisma/engines-version', { prisma: { enginesVersion: HASH } }); + return dir; +} + +/** binaries.prisma.sh, recorded: each file's gzip and its sha256. */ +function server(files: Record, tamper = false) { + const asked: string[] = []; + const fetcher = (async (url: string) => { + asked.push(url); + const name = url.replace(`https://binaries.prisma.sh/all_commits/${HASH}/${TARGET}/`, ''); + const file = name.replace(/\.gz(\.sha256)?$/, ''); + if (!(file in files)) return new Response('', { status: 404 }); + const gz = gzipSync(files[file] as string); + const body = name.endsWith('.sha256') + ? `${createHash('sha256') + .update(tamper ? 'other' : gz) + .digest('hex')} ${file}.gz\n` + : gz; + return new Response(body); + }) as typeof fetch; + return { fetcher, asked }; +} + +describe('prismaEngineEnv (UPT-95)', () => { + const made: string[] = []; + const temp = (dir: string) => { + made.push(dir); + return dir; + }; + afterEach(() => { + for (const dir of made.splice(0)) rmSync(dir, { recursive: true, force: true }); + }); + + it('fetches the schema engine Prisma 7 names, verified, and points Prisma at it', async () => { + const store = temp(mkdtempSync(join(tmpdir(), 'uptide-store-'))); + const { fetcher, asked } = server({ 'schema-engine': 'engine 7' }); + const env = await prismaEngineEnv([temp(project('7.10.0'))], store, undefined, fetcher, TARGET); + expect(env).toEqual({ + PRISMA_SCHEMA_ENGINE_BINARY: join(store, HASH, TARGET, 'schema-engine'), + }); + expect(readFileSync(env.PRISMA_SCHEMA_ENGINE_BINARY as string, 'utf8')).toBe('engine 7'); + // Kept: the next repository on the same engines fetches nothing. + asked.length = 0; + await prismaEngineEnv([temp(project('7.10.0'))], store, undefined, fetcher, TARGET); + expect(asked).toEqual([]); + }); + + it('adds the query engine library for Prisma 5 and 6', async () => { + const store = temp(mkdtempSync(join(tmpdir(), 'uptide-store-'))); + const { fetcher } = server({ 'schema-engine': 's', 'libquery_engine.so.node': 'q' }); + const env = await prismaEngineEnv([temp(project('6.19.0'))], store, undefined, fetcher, TARGET); + expect(Object.keys(env).sort()).toEqual([ + 'PRISMA_QUERY_ENGINE_LIBRARY', + 'PRISMA_SCHEMA_ENGINE_BINARY', + ]); + }); + + it('sets nothing when a download does not match its sha256, or there is no Prisma', async () => { + const store = temp(mkdtempSync(join(tmpdir(), 'uptide-store-'))); + const lines: string[] = []; + const { fetcher } = server({ 'schema-engine': 'engine' }, true); + const log = (l: string) => lines.push(l); + expect(await prismaEngineEnv([temp(project('7.10.0'))], store, log, fetcher, TARGET)).toEqual( + {}, + ); + expect(lines.join('\n')).toMatch(/sha256 does not match/); + const none = temp(mkdtempSync(join(tmpdir(), 'uptide-engines-'))); + expect(await prismaEngineEnv([none], store, undefined, fetcher, TARGET)).toEqual({}); + }); +}); + +it('names the binary target Prisma downloads for', () => { + const ubuntu = () => 'ID=ubuntu\nID_LIKE=debian\n'; + expect(binaryTarget('linux', 'x64', () => '3.0.x', ubuntu)).toBe('debian-openssl-3.0.x'); + expect(binaryTarget('linux', 'arm64', () => '3.0.x', ubuntu)).toBe('linux-arm64-openssl-3.0.x'); + expect( + binaryTarget( + 'linux', + 'x64', + () => '1.1.x', + () => 'ID="rhel"\n', + ), + ).toBe('rhel-openssl-1.1.x'); + expect( + binaryTarget( + 'linux', + 'x64', + () => '3.0.x', + () => 'ID=alpine\n', + ), + ).toBe('linux-musl-openssl-3.0.x'); + expect(binaryTarget('darwin', 'arm64')).toBe('darwin-arm64'); + expect(binaryTarget('linux', 'x64', () => undefined, ubuntu)).toBeUndefined(); +}); diff --git a/packages/core/src/packs/tooling/prisma-engines.ts b/packages/core/src/packs/tooling/prisma-engines.ts new file mode 100644 index 00000000..65e66893 --- /dev/null +++ b/packages/core/src/packs/tooling/prisma-engines.ts @@ -0,0 +1,151 @@ +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + chmodSync, + existsSync, + mkdirSync, + readFileSync, + realpathSync, + renameSync, + writeFileSync, +} from 'node:fs'; +import { dirname, join } from 'node:path'; +import { gunzipSync } from 'node:zlib'; + +/** + * Prisma's CLI downloads its engines on first use when the install script did not run, and + * ground truth keeps generators off the network (UPT-95). So before any repository code runs, + * uptide fetches the engines the locked Prisma names itself: the commit from + * `@prisma/engines-version` (read as data, never executed), the files from binaries.prisma.sh, + * checked against their published sha256. Prisma then uses them through its documented + * `PRISMA_SCHEMA_ENGINE_BINARY` and `PRISMA_QUERY_ENGINE_LIBRARY` variables. Prisma 7 needs the + * schema engine only; 5 and 6 also the query engine library. Nothing fetched (no Prisma, an + * older one, an unknown platform, a failed download) means no variables: the generator then + * fails on the blocked mirror, and `pack test` fails that repository. + */ +export async function prismaEngineEnv( + /** The installed project and its workspaces: where `node_modules/prisma` may be. */ + dirs: string[], + /** Where fetched engines are kept, shared by every repository. */ + store: string, + log?: (line: string) => void, + fetcher: typeof fetch = fetch, + target = binaryTarget(), +): Promise> { + const locked = new Map(); + for (const dir of dirs) { + const prisma = findPackage(dir, 'prisma'); + if (!prisma) continue; + const major = Number(readJson(join(prisma, 'package.json'))?.version?.split('.')[0]); + const engines = findPackage(prisma, '@prisma/engines'); + const version = engines && findPackage(engines, '@prisma/engines-version'); + const hash = version && readJson(join(version, 'package.json'))?.prisma?.enginesVersion; + if (major >= 5 && hash && /^[0-9a-f]{40}$/.test(hash)) locked.set(hash, major); + } + // ponytail: one Prisma per repository; two versions would need per-workspace variables. + if (locked.size !== 1 || !target) return {}; + const [[hash, major]] = [...locked] as [[string, number]]; + const files: [string, string][] = [['PRISMA_SCHEMA_ENGINE_BINARY', schemaEngine(target)]]; + if (major < 7) files.push(['PRISMA_QUERY_ENGINE_LIBRARY', queryLibrary(target)]); + const env: Record = {}; + for (const [name, file] of files) { + const path = join(store, hash, target, file); + if (!existsSync(path)) { + try { + await download( + `https://binaries.prisma.sh/all_commits/${hash}/${target}/${file}.gz`, + path, + fetcher, + ); + log?.(`fetched prisma engine ${file} (${hash.slice(0, 12)}, ${target})`); + } catch (err) { + log?.(`prisma engine ${file} not fetched: ${(err as Error).message}`); + return {}; + } + } + env[name] = path; + } + return env; +} + +async function download(url: string, path: string, fetcher: typeof fetch): Promise { + const get = async (u: string): Promise => { + const res = await fetcher(u, { signal: AbortSignal.timeout(120_000) }); + if (!res.ok) throw new Error(`${u}: HTTP ${res.status}`); + return Buffer.from(await res.arrayBuffer()); + }; + const [gz, sum] = await Promise.all([get(url), get(`${url}.sha256`)]); + const expected = sum.toString('utf8').trim().split(/\s+/)[0]; + if (createHash('sha256').update(gz).digest('hex') !== expected) + throw new Error(`${url}: sha256 does not match`); + mkdirSync(dirname(path), { recursive: true }); + const partial = `${path}.partial-${process.pid}`; + writeFileSync(partial, gunzipSync(gz)); + chmodSync(partial, 0o755); + renameSync(partial, path); +} + +/** Node's lookup of `name` from `from`: each ancestor's `node_modules`, real paths (pnpm). */ +function findPackage(from: string, name: string): string | undefined { + for (let at = realpathSync(from); ; at = dirname(at)) { + const dir = join(at, 'node_modules', name); + if (existsSync(join(dir, 'package.json'))) return realpathSync(dir); + if (dirname(at) === at) return undefined; + } +} + +function readJson( + path: string, +): { version?: string; prisma?: { enginesVersion?: string } } | undefined { + try { + return JSON.parse(readFileSync(path, 'utf8')); + } catch { + return undefined; + } +} + +const schemaEngine = (target: string): string => + target === 'windows' ? 'schema-engine.exe' : 'schema-engine'; + +function queryLibrary(target: string): string { + if (target === 'windows') return 'query_engine.dll.node'; + return target.startsWith('darwin') ? 'libquery_engine.dylib.node' : 'libquery_engine.so.node'; +} + +/** + * Prisma's binary target for this machine (`debian-openssl-3.0.x`, `darwin-arm64`), as its + * `@prisma/get-platform` names it on the platforms CI and contributors run; undefined where + * it cannot tell, and then nothing is fetched. + */ +export function binaryTarget( + platform: string = process.platform, + arch: string = process.arch, + openssl: () => string | undefined = opensslVersion, + osRelease: () => string = () => readFileSync('/etc/os-release', 'utf8'), +): string | undefined { + if (platform === 'darwin') return arch === 'arm64' ? 'darwin-arm64' : 'darwin'; + if (platform === 'win32') return 'windows'; + if (platform !== 'linux') return undefined; + const ssl = openssl(); + if (!ssl) return undefined; + let ids = ''; + try { + ids = (osRelease().match(/^ID(?:_LIKE)?=.*$/gm) ?? []).join(' '); + } catch {} + if (/alpine/.test(ids)) + return arch === 'arm64' ? `linux-musl-arm64-openssl-${ssl}` : `linux-musl-openssl-${ssl}`; + if (arch === 'arm64') return `linux-arm64-openssl-${ssl}`; + return /rhel|centos|fedora/.test(ids) ? `rhel-openssl-${ssl}` : `debian-openssl-${ssl}`; +} + +/** `OpenSSL 3.0.13 …` → `3.0.x`, `OpenSSL 1.1.1w …` → `1.1.x`. */ +function opensslVersion(): string | undefined { + try { + const out = execFileSync('openssl', ['version'], { encoding: 'utf8', timeout: 10_000 }); + const m = /OpenSSL (\d+)\.(\d+)/.exec(out); + if (!m) return undefined; + return m[1] === '1' ? `1.${m[2]}.x` : `${m[1]}.0.x`; + } catch { + return undefined; + } +} diff --git a/packages/core/src/packs/tooling/truth.test.ts b/packages/core/src/packs/tooling/truth.test.ts index 28010def..40eae35e 100644 --- a/packages/core/src/packs/tooling/truth.test.ts +++ b/packages/core/src/packs/tooling/truth.test.ts @@ -1,9 +1,9 @@ -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; import type { GroundTruthRepo } from '../contract.js'; -import { cachedRepoDir, ensureRepo, pinnedRepo } from './truth.js'; +import { cachedRepoDir, ensureRepo, generatorFailures, pinnedRepo } from './truth.js'; // Synthetic: a project in `app/` of a monorepo, the shape that made packs:fetch install at // the root (UPT-61). @@ -27,7 +27,7 @@ describe('pinnedRepo', () => { cacheDir = mkdtempSync(join(tmpdir(), 'uptide-truth-')); const dir = cachedRepoDir(pinned, cacheDir); - mkdirSync(dir, { recursive: true }); + mkdirSync(join(dir, 'app'), { recursive: true }); const marker = (installed: string[]) => writeFileSync( join(dir, '.uptide-ground-truth.json'), @@ -50,3 +50,35 @@ describe('pinnedRepo', () => { }); }); }); + +describe('generators (UPT-95)', () => { + let cacheDir: string | undefined; + afterEach(() => cacheDir && rmSync(cacheDir, { recursive: true, force: true })); + + it('runs them on a cached install, again while one fails, and reports it', async () => { + const pinned = { repo: 'example/prisma', commit: 'b'.repeat(40) }; + cacheDir = mkdtempSync(join(tmpdir(), 'uptide-truth-')); + const dir = cachedRepoDir(pinned, cacheDir); + mkdirSync(join(dir, 'prisma'), { recursive: true }); + mkdirSync(join(dir, 'node_modules/.bin'), { recursive: true }); + writeFileSync(join(dir, 'package.json'), '{"name":"app"}'); + writeFileSync(join(dir, 'prisma/schema.prisma'), 'generator client { provider = "x" }'); + // A stand-in binary: counts its runs, then fails the way a broken schema does. + const bin = join(dir, 'node_modules/.bin/prisma'); + writeFileSync(bin, `#!/bin/sh\necho run >> "${dir}/runs"\necho "schema invalid" >&2\nexit 1\n`); + chmodSync(bin, 0o755); + // Installed before generators were part of the preparation: no fetch, no install. + writeFileSync( + join(dir, '.uptide-ground-truth.json'), + JSON.stringify({ ...pinned, installed: ['.'] }), + ); + + await expect(ensureRepo(pinned, { cacheDir, offline: true })).resolves.toBe(dir); + await ensureRepo(pinned, { cacheDir, offline: true }); + // A failure is not final: a missing engine is fetched on the next run, so it runs again. + expect(readFileSync(join(dir, 'runs'), 'utf8')).toBe('run\nrun\n'); + expect(generatorFailures(pinned, cacheDir)).toEqual([ + 'generator failed: prisma generate --schema prisma/schema.prisma (schema invalid)', + ]); + }); +}); diff --git a/packages/core/src/packs/tooling/truth.ts b/packages/core/src/packs/tooling/truth.ts index f6ce5728..6e1221e6 100644 --- a/packages/core/src/packs/tooling/truth.ts +++ b/packages/core/src/packs/tooling/truth.ts @@ -2,15 +2,19 @@ import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync import { join } from 'node:path'; import { defaultCacheDir } from '../../cache/paths.js'; import { UptideError } from '../../errors.js'; +import { type Generated, generateClients } from '../../fix/generate.js'; import { packageManager } from '../../fix/managers/manager.js'; import { command, git } from '../../fix/process.js'; +import { workspacePackagesOrRoot } from '../../workspaces.js'; import type { GroundTruth, GroundTruthRepo } from '../contract.js'; +import { prismaEngineEnv } from './prisma-engines.js'; /** * Ground-truth repositories are fetched once and kept: a shallow fetch of the one pinned - * commit, dependencies installed with lifecycle scripts off, nothing from the repository ever - * executed (no hook, no script; the same environment `fix` installs with). After the first - * run `uptide pack test` needs no network but the registry tarballs `check` caches itself. + * commit, dependencies installed with lifecycle scripts off (no hook, no script; the same + * environment `fix` installs with), then the repository's known code generators, as `fix` + * runs them (a Prisma client: a user's checkout has it). After the first run `uptide pack + * test` needs no network but the registry tarballs `check` caches itself. */ export function truthCacheDir(env: NodeJS.ProcessEnv = process.env): string { return env.UPTIDE_GROUND_TRUTH_CACHE || join(defaultCacheDir(env), 'ground-truth'); @@ -46,6 +50,8 @@ export function truthProblems(truth: GroundTruth, packageName: string): string[] if (r.directory && (r.directory.startsWith('/') || /(^|\/)\.\.(\/|$)/.test(r.directory))) problems.push(`${where}: directory is relative and inside the repository`); if (!r.from || !r.to) problems.push(`${where}: from and to are exact versions`); + if (r.withoutGenerated !== undefined && !r.withoutGenerated.trim()) + problems.push(`${where}: withoutGenerated says why the generators cannot run`); for (const f of r.findings ?? []) if (!f.file || !Number.isInteger(f.line) || f.line < 1 || !f.rule) problems.push(`${where}: every finding has a file, a line and a rule`); @@ -137,10 +143,21 @@ interface Marker { commit: string; /** The directories installed so far (`.` for the root). */ installed: string[]; + /** What the code generators did after each directory's install (UPT-95). */ + generated?: Record; } const MARKER = '.uptide-ground-truth.json'; +/** + * Prisma downloads its engine binaries on first use when the install script did not run (5 + * and 6 the schema engine and query library, 7 the schema engine): repository code stays off + * the network behind a mirror nothing listens on (UPT-95). The engines it needs are fetched + * before it runs, by uptide (`prisma-engines.ts`); one that is still missing fails the + * generator, and `pack test` fails the repository. + */ +const NO_DOWNLOADS = { PRISMA_ENGINES_MIRROR: 'http://127.0.0.1:9' }; + function readMarker(dir: string): Marker | undefined { try { return JSON.parse(readFileSync(join(dir, MARKER), 'utf8')) as Marker; @@ -174,9 +191,13 @@ export async function ensureRepo(entry: PinnedRepo, options: EnsureOptions = {}) const cacheDir = options.cacheDir ?? truthCacheDir(); const dir = cachedRepoDir(entry, cacheDir); const install = options.install ?? true; + const at = entry.directory ?? '.'; const marker = readMarker(dir); - if (marker && (!install || marker.installed.includes(entry.directory ?? '.'))) return dir; - if (options.offline) + // A cache from before UPT-95 is installed but not generated, or a generator failed there + // (an engine missing then): generating again needs no fetch and no install. + const generatedAll = marker?.generated?.[at]?.every((g) => g.status === 'generated'); + if (marker && (!install || (marker.installed.includes(at) && generatedAll))) return dir; + if (options.offline && !marker?.installed.includes(at)) throw new UptideError( 'GROUND_TRUTH_NOT_CACHED', `${entry.repo}@${entry.commit.slice(0, 12)} is not in the ground-truth cache (${cacheDir}); run without --offline once, or pnpm packs:fetch`, @@ -229,6 +250,7 @@ export async function ensureRepo(entry: PinnedRepo, options: EnsureOptions = {}) } } const installed = [...(marker?.installed ?? [])]; + const generated = { ...marker?.generated }; if (install) { const project = entry.directory ? join(dir, entry.directory) : dir; if (entry.directory && (/(^|\/)\.\.(\/|$)/.test(entry.directory) || !existsSync(project))) @@ -236,19 +258,46 @@ export async function ensureRepo(entry: PinnedRepo, options: EnsureOptions = {}) 'INVALID_GROUND_TRUTH', `no directory ${entry.directory} in ${entry.repo}`, ); - const result = await installDependencies(project, entry.repo, options); - installed.push(entry.directory ?? '.'); - options.log?.( - `installed with ${result.manager}${result.frozen ? '' : ', lockfile not frozen'}`, + if (!installed.includes(at)) { + const result = await installDependencies(project, entry.repo, options); + installed.push(at); + options.log?.( + `installed with ${result.manager}${result.frozen ? '' : ', lockfile not frozen'}`, + ); + } + const workspaces = workspacePackagesOrRoot(project); + const engines = await prismaEngineEnv( + workspaces.map((w) => join(project, w)), + join(cacheDir, 'prisma-engines'), + options.log, + options.offline ? () => Promise.reject(new Error('offline')) : fetch, ); + generated[at] = await generateClients(project, workspaces, { ...NO_DOWNLOADS, ...engines }); + for (const g of generated[at]) + options.log?.(`${g.status}: ${g.command}${g.workspace === '.' ? '' : ` in ${g.workspace}`}`); } - writeFileSync( - join(dir, MARKER), - `${JSON.stringify({ repo: entry.repo, commit: entry.commit, installed })}\n`, - ); + const record: Marker = { repo: entry.repo, commit: entry.commit, installed, generated }; + writeFileSync(join(dir, MARKER), `${JSON.stringify(record)}\n`); return dir; } +/** + * The generators that did not produce their code in a cached repository (failed, timed out, + * refused), as `pack test` reports them: that repository is scored without the code. + */ +export function generatorFailures(entry: PinnedRepo, cacheDir = truthCacheDir()): string[] { + const done = readMarker(cachedRepoDir(entry, cacheDir))?.generated?.[entry.directory ?? '.']; + return (done ?? []) + .filter((g) => g.status !== 'generated') + .map((g) => { + // The last line that says what failed: codegen ends on its summary (`✔ Generate outputs`). + const lines = g.output?.trim().split('\n') ?? []; + const why = lines.findLast((l) => /fail|error|cannot/i.test(l)) ?? lines.at(-1); + const where = g.workspace === '.' ? '' : ` in ${g.workspace}`; + return `generator ${g.status === 'timeout' ? 'timed out' : 'failed'}: ${g.command}${where}${why ? ` (${why})` : ''}`; + }); +} + /** * A fixture repository of this tree, installed in place (its `node_modules` is ignored by * git). Fixtures are synthetic: they are scored, never counted toward `verified`. diff --git a/packages/core/src/packs/typescript/ground-truth.json b/packages/core/src/packs/typescript/ground-truth.json index d70ac0cf..0b9509c4 100644 --- a/packages/core/src/packs/typescript/ground-truth.json +++ b/packages/core/src/packs/typescript/ground-truth.json @@ -9,6 +9,7 @@ "directory": "client", "from": "5.6.2", "to": "6.0.3", + "withoutGenerated": "graphql-codegen fails at this commit: codegen.ts names no schema", "why": "baseUrl \"src\" with ten paths entries: removed, paths rewritten relative (\"./src/graphql/*\")", "findings": [ { diff --git a/packages/core/src/packs/typescript/verification.json b/packages/core/src/packs/typescript/verification.json index f4f17dc7..8d960f6a 100644 --- a/packages/core/src/packs/typescript/verification.json +++ b/packages/core/src/packs/typescript/verification.json @@ -1,6 +1,6 @@ { "status": "verified", - "truth": "sha256-b463fcdce83820c229ab46ab51a2afb62c3da5f4f5e6a2b53fee63f302755e68", + "truth": "sha256-74dcdbebdc3dde9432ece913a25ed78e6f7ff2236fa689d9ec592e21e9ebfc43", "repos": 7, "breaking": { "precision": 1, diff --git a/packages/core/src/packs/zod/verification.json b/packages/core/src/packs/zod/verification.json index 15b53076..ce92244d 100644 --- a/packages/core/src/packs/zod/verification.json +++ b/packages/core/src/packs/zod/verification.json @@ -4,10 +4,10 @@ "repos": 5, "breaking": { "precision": 1, - "recall": 0.7639, + "recall": 0.7925, "falsePositives": 0, - "truePositives": 55, - "falseNegatives": 17 + "truePositives": 126, + "falseNegatives": 33 }, "behavior": { "precision": 1, @@ -18,8 +18,8 @@ }, "class": "breaking", "breakingSites": 219, - "breakingFoundAny": 232, - "breakingFound": 55, + "breakingFoundAny": 216, + "breakingFound": 117, "breakingScored": 249, "behaviorSites": 0, "unexercised": ["error-members"], @@ -33,7 +33,7 @@ "falsePositives": 0 }, "string-format": { - "truePositives": 19, + "truePositives": 21, "falsePositives": 0 }, "error-members": { @@ -41,7 +41,7 @@ "falsePositives": 0 }, "error-issues": { - "truePositives": 52, + "truePositives": 51, "falsePositives": 0 } }