From f3fb6a30c6372e7ebccd7a99c679a0147003f8c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 06:47:08 +0000 Subject: [PATCH 1/9] refactor(packs): a public pack contract and registry; zod and stripe implement it Pack (meta with sources and maintainer, rules that claim check's findings and rewrite one site, behavior notes with how they are reported, agent instructions) on top of MigrationPack. check, fix, list and verify take their packs from a generated registry, verified ones only. Zod's rules carry their own rewrite; stripe declares the rules its evidence classifies. Outputs on the storefront fixture and the corpus unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01B2T8GmEVnZwfNrugajEMqi Signed-off-by: Claude --- biome.json | 3 +- packages/core/src/check/check.ts | 42 ++- packages/core/src/fetch/range.ts | 5 + packages/core/src/fetch/semver.d.ts | 2 + packages/core/src/fix/plan.ts | 13 +- packages/core/src/fix/reverify.ts | 6 +- packages/core/src/fix/run.ts | 10 +- packages/core/src/list/list.ts | 5 +- packages/core/src/packs/contract.test.ts | 251 +++++++++++++ packages/core/src/packs/contract.ts | 349 ++++++++++++++++++ packages/core/src/packs/index.ts | 25 ++ packages/core/src/packs/registry.ts | 12 + .../core/src/packs/stripe/ground-truth.json | 34 ++ packages/core/src/packs/stripe/index.ts | 70 +++- .../core/src/packs/stripe/verification.json | 6 + .../core/src/packs/tooling/registry-file.ts | 62 ++++ .../packs/zod/fixtures/error-params/after.ts | 16 + .../packs/zod/fixtures/error-params/before.ts | 16 + .../packs/zod/fixtures/string-format/after.ts | 10 + .../zod/fixtures/string-format/before.ts | 10 + packages/core/src/packs/zod/ground-truth.json | 204 ++++++++++ packages/core/src/packs/zod/index.ts | 78 +++- packages/core/src/packs/zod/verification.json | 6 + packages/core/tsconfig.json | 3 +- 24 files changed, 1183 insertions(+), 55 deletions(-) create mode 100644 packages/core/src/packs/contract.test.ts create mode 100644 packages/core/src/packs/contract.ts create mode 100644 packages/core/src/packs/index.ts create mode 100644 packages/core/src/packs/registry.ts create mode 100644 packages/core/src/packs/stripe/ground-truth.json create mode 100644 packages/core/src/packs/stripe/verification.json create mode 100644 packages/core/src/packs/tooling/registry-file.ts create mode 100644 packages/core/src/packs/zod/fixtures/error-params/after.ts create mode 100644 packages/core/src/packs/zod/fixtures/error-params/before.ts create mode 100644 packages/core/src/packs/zod/fixtures/string-format/after.ts create mode 100644 packages/core/src/packs/zod/fixtures/string-format/before.ts create mode 100644 packages/core/src/packs/zod/ground-truth.json create mode 100644 packages/core/src/packs/zod/verification.json diff --git a/biome.json b/biome.json index 244eed1..72c2006 100644 --- a/biome.json +++ b/biome.json @@ -14,7 +14,8 @@ "!fixtures/**/*.d.ts", "!fixtures/**/expected-*.json", "!fixtures/snapshots", - "!**/__snapshots__" + "!**/__snapshots__", + "!packages/core/src/packs/*/fixtures" ] }, "formatter": { diff --git a/packages/core/src/check/check.ts b/packages/core/src/check/check.ts index ecad04a..914b0d2 100644 --- a/packages/core/src/check/check.ts +++ b/packages/core/src/check/check.ts @@ -30,9 +30,8 @@ import { maxSatisfying } from '../fetch/range.js'; import { planPackage } from '../fix/plan.js'; import { scanImports } from '../list/scan.js'; import { packFixability } from '../packs/fixability.js'; -import { stripePack } from '../packs/stripe/index.js'; +import { activePacks } from '../packs/index.js'; import type { MigrationPack } from '../packs/types.js'; -import { zodPack } from '../packs/zod/index.js'; import { diffRuntime, probeRuntime } from '../runtime/runtime.js'; import { resetSharedState } from '../shared-state.js'; import { requireFindUsages } from './capabilities.js'; @@ -59,8 +58,13 @@ import { unattributedFindings } from './unattributed.js'; import { verdictOf } from './verdict.js'; import { compareVersions, majorsBehind, parseVersion } from './version.js'; -/** Packs `check` can plan with; the same ones `fix` runs. */ -const PACKS: readonly MigrationPack[] = [zodPack, stripePack]; +/** + * Packs `check` can plan with; the same ones `fix` runs: the verified packs, unless the caller + * names its own (`uptide pack test` scores a candidate with it). + */ +function packsOf(opts: Pick): readonly MigrationPack[] { + return opts.packs ?? activePacks(); +} export interface CheckOptions { onProgress?: ProgressListener; @@ -112,6 +116,12 @@ export interface CheckOptions { hardDeadline?: number; /** Set by `check` for its workspace jobs: the dependencies with something to analyze. */ behind?: string[]; + /** + * The packs to use instead of the verified ones (`uptide pack test` passes a candidate). + * Packs are code, which no worker receives: with this set, workspaces are checked in this + * thread. + */ + packs?: readonly MigrationPack[]; } /** What a worker needs to check one workspace; everything is plain data. */ @@ -123,7 +133,7 @@ export interface WorkspaceJob { installedByWorkspace: Record>; /** Packages each workspace's own sources import, declared or not (a text scan). */ importedByWorkspace: Record; - opts: Omit; + opts: Omit; } export interface CheckResult extends CheckReport { @@ -208,7 +218,7 @@ export async function check(opts: CheckOptions): Promise { }; }), ); - const { adapter: _a, fetcher: _f, cache: _c, onProgress: _p, ...givenOpts } = opts; + const { adapter: _a, fetcher: _f, cache: _c, onProgress: _p, packs: _k, ...givenOpts } = opts; const plainOpts: WorkspaceJob['opts'] = { ...givenOpts, // Compiler overlays share a workspace program; serialize them within its memory slot. @@ -251,7 +261,10 @@ export async function check(opts: CheckOptions): Promise { opts: plainOpts, })); const injected = - opts.adapter !== undefined || opts.fetcher !== undefined || opts.cache !== undefined; + opts.adapter !== undefined || + opts.fetcher !== undefined || + opts.cache !== undefined || + opts.packs !== undefined; const estimates = new Map( jobs.map((job) => [ job.workspace, @@ -343,7 +356,10 @@ export async function check(opts: CheckOptions): Promise { // Pack rules preview their actual edit; check and fix must promise the same work. if (adapter.id === 'typescript') for (const report of results.flat()) { - if (report.name !== 'zod') continue; + // Packs whose rules carry their own rewrite (zod, and every pack built on the contract). + const pack = packsOf(opts).find((p) => p.name === report.name); + if (!pack || !(pack.rules as readonly { rewrite?: unknown }[]).some((r) => r.rewrite)) + continue; const sources = new Map(); report.findings = report.findings.map((finding) => { const file = resolve(opts.cwd, report.workspace, finding.usage.file); @@ -356,7 +372,7 @@ export async function check(opts: CheckOptions): Promise { } sources.set(file, source); } - return packFixability(finding, source, zodPack); + return packFixability(finding, source, pack); }); } const order = new Map(workspaces.map((w, i) => [w, i])); @@ -364,7 +380,7 @@ export async function check(opts: CheckOptions): Promise { results.flat().sort((a, b) => (order.get(a.workspace) ?? 0) - (order.get(b.workspace) ?? 0)), catalogByWorkspace, ); - for (const p of packages) p.tier ??= tierOf(PACKS, p.name, p.installed, p.target); + for (const p of packages) p.tier ??= tierOf(packsOf(opts), p.name, p.installed, p.target); for (const p of packages) groupRootCauses(p); if (adapter.id === 'typescript' && opts.plan !== false) planPackages(packages, opts); else for (const p of packages) delete p.planContext; @@ -398,7 +414,7 @@ function planPackages(packages: PackageReport[], opts: CheckOptions): void { }; for (const p of packages) { if (!['breaking', 'deprecated'].includes(p.status) && p.findings.length === 0) continue; - const pack = PACKS.find((candidate) => candidate.name === p.name); + const pack = packsOf(opts).find((candidate) => candidate.name === p.name); const plan = planPackage(p, pack, read, p.planContext ?? {}); if (plan.length === 0) continue; p.plan = plan; @@ -1774,7 +1790,7 @@ async function checkGroup( findings.length = 0; findings.push(...arbitrated); // A pack can see what no type diff shows: a client the SDK bump reconfigures at runtime. - const pack = PACKS.find((candidate) => candidate.name === name); + const pack = packsOf(ctx.opts).find((candidate) => candidate.name === name); const only = prepared.length === 1 ? prepared[0] : undefined; const packDirs = (p: Prepared) => { const target = fetched.find((f) => f.p === p); @@ -1807,7 +1823,7 @@ async function checkGroup( } } // Without a pack, only what the compiler or the runtime probe confirms is called breaking. - const tier = tierOf(PACKS, name, installed[0] as string, targets.at(-1) as string); + const tier = tierOf(packsOf(ctx.opts), name, installed[0] as string, targets.at(-1) as string); const evidence = (f: Finding): Finding['evidence'] => { const p = runtimeOf(f); return evidenceOf(f, p?.runtime, p ? loadRootOf(p) : undefined); diff --git a/packages/core/src/fetch/range.ts b/packages/core/src/fetch/range.ts index 5aa5a9f..bfe5cca 100644 --- a/packages/core/src/fetch/range.ts +++ b/packages/core/src/fetch/range.ts @@ -8,3 +8,8 @@ export function satisfies(version: string, range: string): boolean { export function maxSatisfying(versions: readonly string[], range: string): string | undefined { return semver.maxSatisfying([...versions], range) ?? undefined; } + +/** The lowest version a range admits (`>=4 <5` → `4.0.0`), or undefined for no valid range. */ +export function minVersion(range: string): string | undefined { + return semver.validRange(range) ? (semver.minVersion(range)?.version ?? undefined) : undefined; +} diff --git a/packages/core/src/fetch/semver.d.ts b/packages/core/src/fetch/semver.d.ts index 90c2575..e3d5937 100644 --- a/packages/core/src/fetch/semver.d.ts +++ b/packages/core/src/fetch/semver.d.ts @@ -2,6 +2,8 @@ declare module 'semver' { const semver: { satisfies(version: string, range: string): boolean; maxSatisfying(versions: string[], range: string): string | null; + minVersion(range: string): { version: string } | null; + validRange(range: string): string | null; }; export default semver; } diff --git a/packages/core/src/fix/plan.ts b/packages/core/src/fix/plan.ts index d40eb4f..83f2a6a 100644 --- a/packages/core/src/fix/plan.ts +++ b/packages/core/src/fix/plan.ts @@ -1,4 +1,5 @@ import type { Finding, PackageReport, PlanGroup } from '../domain/report.js'; +import type { Pack } from '../packs/contract.js'; import type { MigrationPack, PackContext } from '../packs/types.js'; import { changeRule, diagnosticTitle, RULE_TITLES, ruleTitle } from './report.js'; import { selectedFindings } from './select.js'; @@ -90,7 +91,7 @@ export function planPackage( const assisted = sites.filter((s) => s.outcome !== 'mechanical'); const group: PlanGroup = { rule: first.rule, - title: titleOf(first.rule, sites), + title: titleOf(first.rule, sites, usable ? summaryOf(pack, first.rule) : undefined), severity, by: { rule: count('mechanical'), agent: count('agent'), manual: count('manual') }, sites: sites.length, @@ -141,7 +142,13 @@ function written(path: string): string { } /** The PR body's wording when the rule is known; plain English for everything else. */ -function titleOf(rule: string, sites: Planned[]): string { +/** A contract pack's own words for one of its rules or notes (`summary`). */ +function summaryOf(pack: MigrationPack, rule: string): string | undefined { + const notes = 'behavior' in pack ? (pack as Pack).behavior : []; + const rules = pack.rules as readonly { id: string; summary?: string }[]; + return [...rules, ...notes].find((r) => r.id === rule)?.summary; +} +function titleOf(rule: string, sites: Planned[], summary?: string): string { if (rule === 'typescript-no-js-api') return 'TypeScript 7 has no JavaScript compiler API in its main entry'; const evidence = sites @@ -150,7 +157,7 @@ function titleOf(rule: string, sites: Planned[]): string { `${s.finding.change.path} ${s.finding.usage.compileError ?? ''} ${s.finding.usage.snippet}`, ) .join(' '); - const known = RULE_TITLES[rule]?.plain ?? ruleTitle(rule, /ZodTypeDef/.test(evidence)); + const known = RULE_TITLES[rule]?.plain ?? ruleTitle(rule, /ZodTypeDef/.test(evidence)) ?? summary; if (known) return known.replaceAll('`', ''); const first = (sites[0] as Planned).finding; const code = /^TS(\d+)$/.exec(rule); diff --git a/packages/core/src/fix/reverify.ts b/packages/core/src/fix/reverify.ts index f557198..71ce152 100644 --- a/packages/core/src/fix/reverify.ts +++ b/packages/core/src/fix/reverify.ts @@ -3,9 +3,8 @@ import { dirname, join, resolve } from 'node:path'; import { workspacePackagesOf } from '../adapters/typescript/repo.js'; import type { ProgressListener } from '../domain/progress.js'; import { UptideError } from '../errors.js'; -import { stripePack } from '../packs/stripe/index.js'; +import { activePack } from '../packs/index.js'; import type { MigrationPack, PackContext } from '../packs/types.js'; -import { zodPack } from '../packs/zod/index.js'; import { uptideVersionInfo } from '../version.js'; import { git } from './process.js'; import { prBody } from './report.js'; @@ -79,8 +78,7 @@ export async function reverify( .filter((line) => line.trim() !== '' && !line.slice(3).startsWith('.uptide/')); if (pending.length) throw new UptideError('DIRTY_WORKING_TREE', 'uptide verify requires a clean working tree'); - const pack = - options.pack ?? [zodPack, stripePack].find((candidate) => candidate.name === report.package); + const pack = options.pack ?? activePack(report.package); if (!pack) throw new Error(`no migration pack for ${report.package}`); const workspaces = workspacePackagesOf(root).filter((w) => { const p = JSON.parse(readFileSync(join(root, w, 'package.json'), 'utf8')); diff --git a/packages/core/src/fix/run.ts b/packages/core/src/fix/run.ts index 69b47f5..a7e93c2 100644 --- a/packages/core/src/fix/run.ts +++ b/packages/core/src/fix/run.ts @@ -9,10 +9,9 @@ import { UptideError } from '../errors.js'; import { ACCEPTED_KEYS, selectLlm } from '../llm/config.js'; import { DEFAULT_MAX_COST_USD, providerFixer } from '../llm/fixer.js'; import { genericPack } from '../packs/generic.js'; -import { stripePack } from '../packs/stripe/index.js'; +import { activePack } from '../packs/index.js'; import { payloadApiVersions, stripeUsageContext } from '../packs/stripe/relevance.js'; import type { MigrationPack, PackContext } from '../packs/types.js'; -import { zodPack } from '../packs/zod/index.js'; import { resetSharedState } from '../shared-state.js'; import { UPTIDE_COMMAND, uptideVersionInfo } from '../version.js'; import { assist } from './assisted.js'; @@ -160,10 +159,7 @@ async function fixPackage(options: FixOptions, services: FixServices): Promise p.name === options.only) ?? - genericPack(options.only); + let pack = options.pack ?? activePack(options.only) ?? genericPack(options.only); if (pack.name !== options.only) throw new Error(`no migration pack for ${options.only}`); const fixer = options.fixer === null ? undefined : (options.fixer ?? providerFixer(selection)); // Without a pack every edit is the agent's: no agent, nothing this command can do. @@ -579,7 +575,7 @@ export function confirmedEvidence( */ export function refreshReview( result: FixReport, - pack: MigrationPack | undefined = [zodPack, stripePack].find((p) => p.name === result.package), + pack: MigrationPack | undefined = activePack(result.package), ): void { if (!pack || !result.packContext) return; const rules = [...new Set(result.sites.map((s) => s.rule).filter((r): r is string => !!r))]; diff --git a/packages/core/src/list/list.ts b/packages/core/src/list/list.ts index 448a145..5b4cd6a 100644 --- a/packages/core/src/list/list.ts +++ b/packages/core/src/list/list.ts @@ -10,8 +10,7 @@ import type { Tier } from '../domain/report.js'; import { errorCode } from '../errors.js'; import { loadRegistryConfig, registryFor } from '../fetch/npmrc.js'; import { advisoriesEnabled } from '../llm/config.js'; -import { stripePack } from '../packs/stripe/index.js'; -import { zodPack } from '../packs/zod/index.js'; +import { activePacks } from '../packs/index.js'; import type { TaskCommands } from './config.js'; import { BUILD_TOOLS, @@ -514,7 +513,7 @@ export async function listDependencies(opts: ListOptions): Promise { : []), ] : (reasons.get(name) ?? []), - tier: tierOf([zodPack, stripePack], localName(name), current, latest), + tier: tierOf(activePacks(), localName(name), current, latest), workspaces: [...new Set(outdated.flatMap(([, w]) => w))].sort(), usage: { files: scanned?.files.length ?? 0, diff --git a/packages/core/src/packs/contract.test.ts b/packages/core/src/packs/contract.test.ts new file mode 100644 index 0000000..25ca25a --- /dev/null +++ b/packages/core/src/packs/contract.test.ts @@ -0,0 +1,251 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import type { Finding } from '../domain/report.js'; +import { + definePack, + type GroundTruth, + recordedStatus, + replaceAtSite, + statusOf, + truthDigest, +} from './contract.js'; +import { activePacks, registeredPacks } from './index.js'; +import { runFixtures } from './tooling/fixtures.js'; +import { scoreSites } from './tooling/pack-test.js'; +import { parseRegistry, renderRegistry } from './tooling/registry-file.js'; +import { truthProblems } from './tooling/truth.js'; + +const packsDir = fileURLToPath(new URL('.', import.meta.url)); + +describe('every registered pack meets the contract', () => { + it('the registry file is exactly what pack new renders', () => { + const text = readFileSync(join(packsDir, 'registry.ts'), 'utf8'); + expect(renderRegistry(parseRegistry(text))).toBe(text); + expect(parseRegistry(text).map((e) => e.dir)).toEqual(registeredPacks().map((e) => e.dir)); + }); + + it.each(registeredPacks().map((e) => [e.pack.name, e] as const))('%s', (_name, entry) => { + const { pack } = entry; + expect(pack.meta.package).toBe(pack.name); + expect(pack.meta.maintainer).not.toBe(''); + expect(pack.meta.sources.length).toBeGreaterThan(0); + for (const s of pack.meta.sources) expect(s.url).toMatch(/^https:\/\//); + const ids = [...pack.rules.map((r) => r.id), ...pack.behavior.map((b) => b.id)]; + expect(new Set(ids).size).toBe(ids.length); + for (const rule of pack.rules) { + expect(rule.summary, rule.id).not.toBe(''); + expect(['breaking', 'deprecated']).toContain(rule.severity); + } + for (const note of pack.behavior) { + expect(note.reported.length, note.id).toBeGreaterThan(0); + if (note.reported.includes('finding')) expect(note.detect, note.id).toBeDefined(); + } + expect(pack.instructions).not.toBe(''); + + const dir = join(packsDir, entry.dir); + const fixtures = runFixtures(pack, dir); + expect(fixtures.unknown).toEqual([]); + expect(fixtures.rewriteFailures).toEqual([]); + expect(fixtures.falsePositives).toEqual([]); + expect(fixtures.falseNegatives).toEqual([]); + + const file = join(dir, 'ground-truth.json'); + expect(existsSync(file)).toBe(true); + const text = readFileSync(file, 'utf8'); + expect(truthProblems(JSON.parse(text) as GroundTruth, pack.name)).toEqual([]); + // The status the CLI shows was measured on the ground truth as committed. + if (entry.verification.status === 'verified') + expect(entry.verification.truth).toBe(truthDigest(text)); + }); + + it('every rule that rewrites or detects has a fixture site, and one it leaves alone', () => { + for (const { pack, dir } of registeredPacks()) { + const before = (c: string) => + readFileSync(join(packsDir, dir, 'fixtures', c, 'before.ts'), 'utf8'); + const cases = runFixtures(pack, join(packsDir, dir)).cases; + const marked = cases.map(before).join('\n'); + for (const rule of pack.rules.filter((r) => r.rewrite || r.detect)) { + expect(marked, `${pack.name} ${rule.id}`).toMatch( + new RegExp(`@uptide ${rule.id}(?! keep)(?:\\s|$)`), + ); + expect(marked, `${pack.name} ${rule.id} keep`).toContain(`@uptide ${rule.id} keep`); + } + } + }); + + it('check, list and fix use the verified packs only', () => { + const verified = registeredPacks() + .filter((e) => recordedStatus(e.verification) === 'verified') + .map((e) => e.pack.name); + expect(activePacks().map((p) => p.name)).toEqual(verified); + }); +}); + +describe('the verified gate', () => { + const truth = (repos: number, fixtures = 0): GroundTruth => ({ + package: 'x', + repos: [ + ...Array.from({ length: repos }, (_, i) => ({ + repo: `o/r${i}`, + commit: 'a'.repeat(40), + from: '1.0.0', + to: '2.0.0', + why: '', + findings: [], + })), + ...Array.from({ length: fixtures }, () => ({ + fixture: 'fixtures/repos/x', + from: '1.0.0', + to: '2.0.0', + why: '', + findings: [], + })), + ], + }); + it('needs two public repositories, no breaking false positive, and something found', () => { + expect(statusOf(truth(2), { falsePositives: 0, predicted: 3 })).toBe('verified'); + expect(statusOf(truth(1, 3), { falsePositives: 0, predicted: 3 })).toBe('candidate'); + expect(statusOf(truth(3), { falsePositives: 1, predicted: 3 })).toBe('candidate'); + expect(statusOf(truth(2), { falsePositives: 0, predicted: 0 })).toBe('candidate'); + }); + it('a record that contradicts itself is a candidate', () => { + const record = { + status: 'verified' as const, + truth: 'sha256-x', + repos: 2, + breaking: { precision: 1, recall: 1, falsePositives: 0 }, + }; + expect(recordedStatus(record)).toBe('verified'); + expect(recordedStatus({ ...record, repos: 1 })).toBe('candidate'); + expect(recordedStatus({ ...record, breaking: { ...record.breaking, falsePositives: 2 } })).toBe( + 'candidate', + ); + expect(recordedStatus(undefined)).toBe('candidate'); + }); +}); + +describe('definePack', () => { + const finding = (line: number, column: number, rule?: string): Finding => ({ + change: { + package: 'toy', + from: '1.2.0', + to: '2.0.0', + path: 'oldName', + kind: 'removed', + severity: 'breaking', + source: 'types', + confidence: 1, + }, + usage: { + file: 'a.ts', + line, + column, + endLine: line, + endColumn: column, + symbolPath: 'oldName', + access: 'call', + snippet: '', + via: 'direct', + }, + severity: 'breaking', + confidence: 1, + fixability: 'mechanical', + reason: '', + ...(rule ? { rule } : {}), + }); + const pack = definePack({ + meta: { + package: 'toy', + from: '>=1 <2', + to: '>=2 <3', + sources: [{ title: 'guide', url: 'https://example.com' }], + maintainer: 'uptide-dev', + }, + rules: [ + { + id: 'rename', + summary: 'oldName is newName', + severity: 'breaking', + kinds: ['removed'], + symbols: /^oldName$/, + guide: 'use newName', + rewrite: (text, f) => replaceAtSite(text, f, 'oldName', 'newName'), + }, + ], + behavior: [ + { + id: 'retry', + summary: 'connect() retries by default', + reported: ['finding'], + detect: (text) => + text.includes('connect(') ? [{ line: 1, column: 1, snippet: 'connect()' }] : [], + }, + ], + instructions: 'only the site', + }); + + it('rewrites only the reported occurrence, within the supported versions', () => { + const text = 'oldName(oldName(1));'; + const out = pack.transform(text, finding(1, 9), { + from: '1.2.0', + to: '2.0.0', + includeDeprecated: false, + }); + expect(out).toMatchObject({ applied: true, rule: 'rename', text: 'oldName(newName(1));' }); + expect( + pack.transform(text, finding(1, 9), { from: '2.0.0', to: '3.0.0', includeDeprecated: true }) + .applied, + ).toBe(false); + expect(replaceAtSite('const oldNameX = 1;', finding(1, 7), 'oldName', 'newName').applied).toBe( + false, + ); + }); + + it("reports a note's sites as the pack's own findings, in files that use the package", () => { + const findings = + pack.runtimeFindings?.({ + root: '/r', + workspace: 'app', + from: '1.2.0', + to: '2.0.0', + installedDir: '', + targetDir: '', + usages: [{ file: 'src/db.ts' } as never, { file: 'src/db.ts' } as never], + read: (file) => (file === join('app', 'src/db.ts') ? 'connect()' : undefined), + }) ?? []; + expect(findings).toHaveLength(1); + expect(findings[0]).toMatchObject({ + rule: 'retry', + severity: 'breaking', + fixability: 'manual', + change: { source: 'pack', path: 'toy:retry' }, + usage: { file: 'src/db.ts', line: 1 }, + }); + }); +}); + +describe('scoring', () => { + it('a site at an expected line under another rule is a hit, listed as such', () => { + const score = scoreSites( + [ + { file: 'a.ts', line: 1, rule: 'x', severity: 'breaking' }, + { file: 'a.ts', line: 2, rule: 'y', severity: 'breaking' }, + { file: 'b.ts', line: 9, rule: 'x', severity: 'breaking' }, + ], + [ + { file: 'a.ts', line: 1, rule: 'x' }, + { file: 'a.ts', line: 2, rule: 'x' }, + { file: 'c.ts', line: 3, rule: 'x' }, + ], + ); + expect(score.falsePositives).toEqual([ + { file: 'b.ts', line: 9, rule: 'x', severity: 'breaking' }, + ]); + expect(score.falseNegatives).toEqual([{ file: 'c.ts', line: 3, rule: 'x' }]); + expect(score.wrongRule).toEqual([ + { file: 'a.ts', line: 2, rule: 'y', severity: 'breaking', expected: 'x' }, + ]); + }); +}); diff --git a/packages/core/src/packs/contract.ts b/packages/core/src/packs/contract.ts new file mode 100644 index 0000000..4a1a904 --- /dev/null +++ b/packages/core/src/packs/contract.ts @@ -0,0 +1,349 @@ +import { createHash } from 'node:crypto'; +import { join } from 'node:path'; +import type { Finding } from '../domain/report.js'; +import type { Usage } from '../domain/usage.js'; +import { satisfies } from '../fetch/range.js'; +import type { MigrationPack, MigrationRule, PackContext, TransformResult } from './types.js'; + +/** + * The public contract of a migration pack (docs/packs.md). `MigrationPack` is what the runner + * calls; `Pack` adds what a person, a reviewer or `uptide pack test` needs to judge it: where + * the knowledge comes from, what each rule claims, what the compiler cannot see, and the real + * repositories it is scored against. + */ + +/** Where the pack's knowledge comes from: the upstream changelog and migration guide. */ +export interface PackSource { + title: string; + url: string; +} + +export interface PackMeta { + /** The npm package name, exactly as a consumer depends on it. */ + package: string; + /** Semver range of the installed versions the pack migrates from (`>=3 <4`). */ + from: string; + /** Semver range of the target versions (`>=4 <5`). */ + to: string; + /** At least one: the changelog or migration guide each rule and note is taken from. */ + sources: readonly PackSource[]; + /** Who answers for the pack: a GitHub handle (`@octocat`) or `uptide-dev`. */ + maintainer: string; +} + +/** One place a rule or a behavior note applies, found in a file's text. */ +export interface SourceSite { + /** 1-based. */ + line: number; + /** 1-based. */ + column: number; + /** The code at the site, trimmed: what the report shows. */ + snippet: string; +} + +/** + * A mechanical rule. Detection is the type diff `check` already does: the rule claims the + * findings whose change kind and symbol path it matches (`kinds`, `symbols`), and `detect` + * adds sites no type diff reports, from a file's text. `rewrite` is the deterministic edit at + * one reported site; a rule without it is assisted, and its `guide` goes to the agent. + */ +export interface PackRule extends MigrationRule { + /** One line, for `pack test`, the docs and the plan. */ + summary: string; + /** What a site of this rule is: it breaks at the target, or it is deprecated there. */ + severity: 'breaking' | 'deprecated'; + /** + * The edit at one reported site, or `{ applied: false, reason }` when the site is not one it + * can migrate safely. Never edits anything but the reported site. + */ + rewrite?(text: string, finding: Finding, context: PackContext): TransformResult; + /** Sites in a file that uses the package which the type diff cannot report. */ + detect?(text: string, file: string): SourceSite[]; +} + +/** + * A change the compiler cannot see: the same code compiles and behaves differently. How a + * site is reported: `finding` lists it in `check` as a pack finding for a person to decide + * (never edited); `decision` lists it under "Decisions for you" in the pull request; + * `test-follow-up` updates a test assertion only after the migrated code made it fail. + */ +export interface BehaviorNote { + id: string; + summary: string; + reported: readonly ('finding' | 'decision' | 'test-follow-up')[]; + /** With `finding`: the sites, from a file that uses the package. */ + detect?(text: string, file: string): SourceSite[]; + /** With `finding`: whether a site breaks at the target, or only needs a look. */ + severity?: 'breaking' | 'deprecated'; +} + +/** A pack as the public contract describes it; see docs/packs.md. */ +export interface Pack extends MigrationPack { + readonly meta: PackMeta; + readonly rules: readonly PackRule[]; + readonly behavior: readonly BehaviorNote[]; + /** What the agent is told for every site no rule rewrites; each rule's `guide` comes first. */ + readonly instructions: string; +} + +/** + * Ground truth: real public repositories at the commit before they made this upgrade, with + * every finding the pack must report there. `fixture` entries are repositories in this tree + * (`fixtures/repos/...`): scored like the others, never counted toward `verified`. + */ +export interface GroundTruthRepo { + /** `owner/name` on GitHub. */ + repo?: string; + /** The full SHA scored: the commit before the upgrade. */ + commit?: string; + /** A repository inside this one, relative to its root, instead of `repo` and `commit`. */ + fixture?: string; + /** Where the expected findings come from: the commit or pull request that made the upgrade. */ + migration?: string; + /** The exact version installed at `commit` and the one to check against. */ + from: string; + to: string; + why: string; + /** Repository-relative `file`, 1-based `line`, and the rule (or behavior note) id. */ + findings: GroundTruthFinding[]; +} + +export interface GroundTruthFinding { + file: string; + line: number; + rule: string; + /** Optional: what changed there in the real upgrade. */ + note?: string; +} + +export interface GroundTruth { + $comment?: string; + package: string; + repos: GroundTruthRepo[]; +} + +/** + * What `uptide pack test --write` recorded: the status the gate below reads at runtime. CI + * runs `pack test` on every pack and fails when this record no longer matches what the + * ground truth says, so the label cannot outlive the evidence behind it. + */ +export interface PackVerification { + status: PackStatus; + /** sha256 of the ground-truth file the result was computed from. */ + truth: string; + /** Public repositories in the ground truth (fixtures excluded). */ + repos: number; + breaking: { precision: number; recall: number; falsePositives: number }; +} + +/** + * `verified`: ground truth from at least two public repositories and no false positive among + * its breaking findings there. Anything else is a `candidate`: it ships, `uptide pack test` + * scores it, and `check`, `list` and `fix` treat the dependency as generic. + */ +export type PackStatus = 'verified' | 'candidate'; + +export const VERIFIED_MIN_REPOS = 2; + +export function publicRepos(truth: GroundTruth): GroundTruthRepo[] { + return truth.repos.filter((r) => r.repo !== undefined && r.commit !== undefined); +} + +/** The gate, from what `pack test` measured. */ +export function statusOf( + truth: GroundTruth, + breaking: { falsePositives: number; predicted: number }, +): PackStatus { + return publicRepos(truth).length >= VERIFIED_MIN_REPOS && + breaking.falsePositives === 0 && + breaking.predicted > 0 + ? 'verified' + : 'candidate'; +} + +/** The digest a verification record is tied to: the ground-truth file as committed. */ +export function truthDigest(text: string): string { + return `sha256-${createHash('sha256').update(text).digest('hex')}`; +} + +/** The gate, at runtime: what the committed record says, and only if it is consistent. */ +export function recordedStatus(record: PackVerification | undefined): PackStatus { + return record?.status === 'verified' && + record.repos >= VERIFIED_MIN_REPOS && + record.breaking.falsePositives === 0 + ? 'verified' + : 'candidate'; +} + +/** A pack in the registry: its code, its directory under `packs/`, and its recorded status. */ +export interface RegisteredPack { + dir: string; + pack: Pack; + verification: PackVerification; +} + +/** The rule a finding belongs to: the one that set it, else the first whose kinds and symbols match. */ +export function ruleFor( + rules: readonly R[], + finding: Finding, +): R | undefined { + return ( + rules.find((r) => r.id === finding.rule) ?? + rules.find((r) => r.kinds.includes(finding.change.kind) && r.symbols.test(finding.change.path)) + ); +} + +/** Rule and note sites found in one file's text, as `check` reports a pack's own finding. */ +function detected( + pack: Pick, + input: { file: string; text: string; from: string; to: string }, +): Finding[] { + const sources = [ + ...pack.rules + .filter((r) => r.detect) + .map((r) => ({ + id: r.id, + summary: r.summary, + severity: r.severity, + detect: r.detect, + manual: !r.rewrite, + })), + ...pack.behavior + .filter((b) => b.detect && b.reported.includes('finding')) + .map((b) => ({ + id: b.id, + summary: b.summary, + severity: b.severity ?? 'breaking', + detect: b.detect, + manual: true, + })), + ]; + const findings: Finding[] = []; + for (const source of sources) + for (const site of source.detect?.(input.text, input.file) ?? []) { + const path = `${pack.meta.package}:${source.id}`; + findings.push({ + change: { + package: pack.meta.package, + from: input.from, + to: input.to, + path, + kind: source.severity === 'deprecated' ? 'deprecated' : 'signature', + severity: source.severity, + source: 'pack', + confidence: 1, + }, + usage: { + file: input.file, + line: site.line, + column: site.column, + endLine: site.line, + endColumn: site.column, + symbolPath: path, + access: 'call', + snippet: site.snippet, + via: 'direct', + } as Usage, + severity: source.severity, + confidence: 1, + fixability: source.manual ? 'manual' : 'mechanical', + reason: source.summary, + rule: source.id, + }); + } + return findings; +} + +export interface PackSpec { + meta: PackMeta; + rules: PackRule[]; + behavior?: BehaviorNote[]; + instructions: string; + /** The version `fix` falls back to when the registry cannot answer. */ + defaultTarget?: string; +} + +/** + * A pack built from the contract alone, which is what `uptide pack new` scaffolds: the runner's + * hooks are derived from the rules and notes. Zod and Stripe implement `Pack` by hand because + * they also gather context no rule has (default messages, API versions). + */ +export function definePack(spec: PackSpec): Pack { + const behavior = spec.behavior ?? []; + const pack: Pack = { + name: spec.meta.package, + meta: spec.meta, + rules: spec.rules, + behavior, + instructions: spec.instructions, + defaultTarget: spec.defaultTarget ?? '', + supports: (from, to) => satisfies(from, spec.meta.from) && satisfies(to, spec.meta.to), + transform(text, finding, context) { + if (!pack.supports(context.from, context.to)) + return { + text, + applied: false, + reason: `pack supports ${spec.meta.package} ${spec.meta.from} to ${spec.meta.to} only`, + }; + const rule = ruleFor(spec.rules, finding); + if (!rule?.rewrite) + return { text, applied: false, reason: 'no mechanical rule for this reported site' }; + if (rule.severity === 'deprecated' && !context.includeDeprecated) + return { text, applied: false, reason: 'deprecated sites need --include-deprecated' }; + const result = rule.rewrite(text, finding, context); + return result.applied ? { ...result, rule: rule.id } : result; + }, + guide(finding) { + const rule = ruleFor(spec.rules, finding); + return [rule?.guide, spec.instructions].filter(Boolean).join('\n'); + }, + reviewNotes: () => [ + ...behavior.map((b) => `${b.summary}`), + ...spec.meta.sources.map((s) => `${s.title}: ${s.url}`), + ], + runtimeFindings({ workspace, from, to, usages, read }) { + const files = [...new Set(usages.map((u) => u.file))].sort(); + return files.flatMap((file) => { + const text = read(join(workspace, file)); + return text === undefined ? [] : detected(pack, { file, text, from, to }); + }); + }, + }; + return pack; +} + +/** Sites detected in one file, for fixtures and ground truth: what `check` would add. */ +export function detectIn( + pack: Pick, + file: string, + text: string, +): { rule: string; line: number }[] { + return detected(pack, { file, text, from: '', to: '' }).map((f) => ({ + rule: f.rule as string, + line: f.usage.line, + })); +} + +/** + * The most common rewrite: one identifier at the reported site becomes another. Only the + * occurrence the finding points at (its line, at or after its column) is replaced, and only + * when it is that whole identifier; anything else is declined with a reason. + */ +export function replaceAtSite( + text: string, + finding: Pick, + before: string, + after: string, +): TransformResult { + const lines = text.split('\n'); + const index = finding.usage.line - 1; + const line = lines[index]; + if (line === undefined) return { text, applied: false, reason: 'the reported line is gone' }; + const word = new RegExp(`(? packStatus(e) === 'verified').map((e) => e.pack); +} + +export function activePack(name: string): MigrationPack | undefined { + return activePacks().find((p) => p.name === name); +} diff --git a/packages/core/src/packs/registry.ts b/packages/core/src/packs/registry.ts new file mode 100644 index 0000000..15a657f --- /dev/null +++ b/packages/core/src/packs/registry.ts @@ -0,0 +1,12 @@ +// Generated by `uptide pack new` (packages/core/src/packs/tooling/registry-file.ts). +// Every pack in this tree, with the status `uptide pack test --write` recorded for it. +import type { PackVerification, RegisteredPack } from './contract.js'; +import { stripePack } from './stripe/index.js'; +import stripeVerification from './stripe/verification.json' with { type: 'json' }; +import { zodPack } from './zod/index.js'; +import zodVerification from './zod/verification.json' with { type: 'json' }; + +export const REGISTRY: readonly RegisteredPack[] = [ + { dir: 'stripe', pack: stripePack, verification: stripeVerification as PackVerification }, + { dir: 'zod', pack: zodPack, verification: zodVerification as PackVerification }, +]; diff --git a/packages/core/src/packs/stripe/ground-truth.json b/packages/core/src/packs/stripe/ground-truth.json new file mode 100644 index 0000000..485ac0c --- /dev/null +++ b/packages/core/src/packs/stripe/ground-truth.json @@ -0,0 +1,34 @@ +{ + "package": "stripe", + "repos": [ + { + "fixture": "fixtures/repos/storefront", + "from": "14.25.0", + "to": "23.0.0", + "why": "synthetic pnpm workspace of this repository (fixtures/truth/storefront.json): apiVersion literal, subscription period, a widened test fixture cast", + "findings": [ + { + "file": "packages/api/src/billing/client.ts", + "line": 9, + "rule": "api-version" + }, + { + "file": "packages/api/src/billing/renewal.test.ts", + "line": 7, + "rule": "fixture-cast" + }, + { + "file": "packages/api/src/billing/renewal.ts", + "line": 12, + "rule": "subscription-period" + }, + { + "file": "packages/api/src/billing/renewal.ts", + "line": 13, + "rule": "subscription-period" + } + ] + } + ], + "$comment": "Ground truth for the stripe pack. Public repositories at the commit before a stripe-node major upgrade, each with every site the real upgrade changed for stripe or the compiler rejected (docs/packs.md, \"Ground truth\"). The storefront fixture is synthetic: scored, never counted toward verified; its sites are the tsc errors of fixtures/truth/storefront.json." +} diff --git a/packages/core/src/packs/stripe/index.ts b/packages/core/src/packs/stripe/index.ts index f534f1a..d9c1911 100644 --- a/packages/core/src/packs/stripe/index.ts +++ b/packages/core/src/packs/stripe/index.ts @@ -5,7 +5,8 @@ import type { Finding } from '../../domain/report.js'; import { createNpmFetcher, releasePackage } from '../../fetch/npm-fetcher.js'; import { satisfies } from '../../fetch/range.js'; import { UPTIDE_COMMAND } from '../../version.js'; -import type { MigrationPack, PackContext, SharedHelper } from '../types.js'; +import type { Pack, PackMeta } from '../contract.js'; +import type { PackContext, SharedHelper } from '../types.js'; import { between, type StripeCatalog } from './changelog.js'; import catalog from './changelog.v1.json' with { type: 'json' }; import { @@ -158,13 +159,33 @@ export function stripeConcerns(findings: Finding[], context: PackContext) { }, }; } -export const stripePack: MigrationPack = { +const SDK_GUIDE = + 'Migrate the reported Stripe SDK usage while preserving payment, subscription and webhook semantics. Use the target diagnostic and checked-in API changelog. Never remove signature verification, change monetary amounts/currency, or introduce a cast to hide incompatible data.'; +const meta: PackMeta = { + package: 'stripe', + from: '>=14', + // Any later version: `supports` also requires the target to be newer than what is installed. + to: '>=14', + sources: [ + { + title: 'stripe-node changelog', + url: 'https://github.com/stripe/stripe-node/blob/master/CHANGELOG.md', + }, + { title: 'Stripe API changelog', url: 'https://docs.stripe.com/changelog' }, + { title: 'Stripe API upgrades', url: 'https://docs.stripe.com/upgrades' }, + ], + maintainer: 'uptide-dev', +}; +export const stripePack: Pack = { name: 'stripe', + meta, defaultTarget: '22.6.2', - supports: (from, to) => satisfies(from, '>=14') && satisfies(to, `>${from}`), + supports: (from, to) => satisfies(from, meta.from) && satisfies(to, `>${from}`), rules: [ { id: 'api-version', + summary: 'an apiVersion literal or LatestApiVersion the target SDK no longer accepts', + severity: 'breaking', kinds: ['type', 'signature', 'narrowed', 'widened', 'required'], symbols: pin, guide: @@ -172,6 +193,8 @@ export const stripePack: MigrationPack = { }, { id: 'api-version-unpinned', + summary: 'a client without apiVersion, whose API version the SDK bump changes at runtime', + severity: 'breaking', kinds: ['type'], symbols: /^Stripe\.StripeConfig#apiVersion$/, guide: @@ -179,6 +202,8 @@ export const stripePack: MigrationPack = { }, { id: 'sdk-surface', + summary: 'any other SDK type or method the target changed; migrated by the agent', + severity: 'breaking', kinds: [ 'removed', 'moved', @@ -190,10 +215,45 @@ export const stripePack: MigrationPack = { 'widened', ], symbols: /./, - guide: - 'Migrate the reported Stripe SDK usage while preserving payment, subscription and webhook semantics. Use the target diagnostic and checked-in API changelog. Never remove signature verification, change monetary amounts/currency, or introduce a cast to hide incompatible data.', + guide: SDK_GUIDE, + }, + // Classified from the site's evidence after the rules above matched (`changeRule` in + // fix/report.ts), never by kind and symbol: listed so the contract names every rule a + // plan or a pull request can show. + { + id: 'subscription-period', + summary: + 'current_period_start/end moved to subscription items (2025-03-31.basil); migrated by the agent through one shared helper, the multi-item case is a decision', + severity: 'breaking', + kinds: [], + symbols: /current_period_(?:end|start)/, + guide: 'See the subscription period guide above.', + }, + { + id: 'fixture-cast', + summary: + 'a test that cast a partial fixture to an SDK type goes through unknown once the types stop overlapping (TS2352)', + severity: 'breaking', + kinds: [], + symbols: /^TS2352$/, + guide: 'Rewritten by rule: the same cast through unknown, in test files only.', + }, + ], + behavior: [ + { + id: 'api-changelog', + summary: + 'API changelog entries between the two pinned API versions, filtered to the methods, fields, parameters and events the code uses', + reported: ['decision'], + }, + { + id: 'webhook-payload-versions', + summary: + 'api_version literals in webhook payload fixtures mirror the endpoint configuration and stay as they are', + reported: ['decision'], }, ], + instructions: SDK_GUIDE, transform: (text, finding, context) => { // A client without apiVersion says the version the target SDK's types describe: with the // bump that is the only honest pin. Staying on the old version is `--pin-current-api`. diff --git a/packages/core/src/packs/stripe/verification.json b/packages/core/src/packs/stripe/verification.json new file mode 100644 index 0000000..d596ce8 --- /dev/null +++ b/packages/core/src/packs/stripe/verification.json @@ -0,0 +1,6 @@ +{ + "status": "verified", + "truth": "", + "repos": 2, + "breaking": { "precision": 1, "recall": 1, "falsePositives": 0 } +} diff --git a/packages/core/src/packs/tooling/registry-file.ts b/packages/core/src/packs/tooling/registry-file.ts new file mode 100644 index 0000000..da7c332 --- /dev/null +++ b/packages/core/src/packs/tooling/registry-file.ts @@ -0,0 +1,62 @@ +/** + * `packs/registry.ts` is generated: `uptide pack new` adds an entry by rendering the whole + * file again from the entries already there, so it is always in the one form biome accepts + * and a hand edit never has to be merged with a generated one. + */ +export interface RegistryEntry { + /** The pack's directory under `packs/`. */ + dir: string; + /** The exported `Pack` constant (`zodPack`). */ + name: string; +} + +const HEADER = `// Generated by \`uptide pack new\` (packages/core/src/packs/tooling/registry-file.ts). +// Every pack in this tree, with the status \`uptide pack test --write\` recorded for it.`; + +export function renderRegistry(entries: readonly RegistryEntry[]): string { + const sorted = [...entries].sort((a, b) => a.dir.localeCompare(b.dir)); + const imports = sorted.flatMap((e) => [ + `import { ${e.name} } from './${e.dir}/index.js';`, + `import ${verificationName(e)} from './${e.dir}/verification.json' with { type: 'json' };`, + ]); + const rows = sorted.map( + (e) => + ` { dir: '${e.dir}', pack: ${e.name}, verification: ${verificationName(e)} as PackVerification },`, + ); + return [ + HEADER, + "import type { PackVerification, RegisteredPack } from './contract.js';", + ...imports, + '', + 'export const REGISTRY: readonly RegisteredPack[] = [', + ...rows, + '];', + '', + ].join('\n'); +} + +function verificationName(e: RegistryEntry): string { + return `${e.name.replace(/Pack$/, '')}Verification`; +} + +/** The entries of a registry file as rendered above. */ +export function parseRegistry(text: string): RegistryEntry[] { + return [...text.matchAll(/\{ dir: '([^']+)', pack: (\w+),/g)].map((m) => ({ + dir: m[1] as string, + name: m[2] as string, + })); +} + +/** `@ai-sdk/react` → `ai-sdk-react`: the directory of a package's pack. */ +export function packDir(name: string): string { + return name.replace(/^@/, '').replace(/[/.]/g, '-').toLowerCase(); +} + +/** `@ai-sdk/react` → `aiSdkReactPack`: the exported constant. */ +export function packConstant(name: string): string { + const words = packDir(name) + .split(/[^a-z0-9]+/) + .filter(Boolean); + const camel = words.map((w, i) => (i === 0 ? w : `${w[0]?.toUpperCase()}${w.slice(1)}`)).join(''); + return `${/^[0-9]/.test(camel) ? `p${camel}` : camel}Pack`; +} diff --git a/packages/core/src/packs/zod/fixtures/error-params/after.ts b/packages/core/src/packs/zod/fixtures/error-params/after.ts new file mode 100644 index 0000000..ab78f94 --- /dev/null +++ b/packages/core/src/packs/zod/fixtures/error-params/after.ts @@ -0,0 +1,16 @@ +import { z } from 'zod'; + +// Literal messages: one callback that tells missing input from invalid input. +export const name = z.string({ error: (iss) => iss.input === undefined ? 'Name is required' : undefined }); // @uptide error-params at:z.string +export const age = z.number({ error: (iss) => iss.input === undefined ? undefined : 'Age must be a number' }); // @uptide error-params at:z.number +export const code = z.string({ /* both */ error: (iss) => iss.input === undefined ? 'A' : 'B', description: 'd' }); // @uptide error-params at:z.string +export const role = z.enum(['admin', 'user'], { error: (iss) => iss.input === undefined ? 'Pick a role' : undefined }); // @uptide error-params at:z.enum + +// Not literals: captured once, in their original order. +const label = (field: string) => `${field} is required`; +export const city = z.string(((options: T) => { const { required_error, invalid_type_error, ...rest } = options; return { ...rest, error: (iss: { input: unknown }) => iss.input === undefined ? required_error : invalid_type_error }; })({ required_error: label('City') })); // @uptide error-params at:z.string + +// Left for a person: an existing errorMap must be merged by hand, a spread may hide the keys. +export const merged = z.string({ required_error: 'x', errorMap: () => ({ message: 'y' }) }); // @uptide error-params keep at:z.string +const base = { description: 'shared' }; +export const spread = z.string({ ...base, required_error: 'x' }); // @uptide error-params keep at:z.string diff --git a/packages/core/src/packs/zod/fixtures/error-params/before.ts b/packages/core/src/packs/zod/fixtures/error-params/before.ts new file mode 100644 index 0000000..67f72b9 --- /dev/null +++ b/packages/core/src/packs/zod/fixtures/error-params/before.ts @@ -0,0 +1,16 @@ +import { z } from 'zod'; + +// Literal messages: one callback that tells missing input from invalid input. +export const name = z.string({ required_error: 'Name is required' }); // @uptide error-params at:z.string +export const age = z.number({ invalid_type_error: 'Age must be a number' }); // @uptide error-params at:z.number +export const code = z.string({ /* both */ required_error: 'A', invalid_type_error: 'B', description: 'd' }); // @uptide error-params at:z.string +export const role = z.enum(['admin', 'user'], { required_error: 'Pick a role' }); // @uptide error-params at:z.enum + +// Not literals: captured once, in their original order. +const label = (field: string) => `${field} is required`; +export const city = z.string({ required_error: label('City') }); // @uptide error-params at:z.string + +// Left for a person: an existing errorMap must be merged by hand, a spread may hide the keys. +export const merged = z.string({ required_error: 'x', errorMap: () => ({ message: 'y' }) }); // @uptide error-params keep at:z.string +const base = { description: 'shared' }; +export const spread = z.string({ ...base, required_error: 'x' }); // @uptide error-params keep at:z.string diff --git a/packages/core/src/packs/zod/fixtures/string-format/after.ts b/packages/core/src/packs/zod/fixtures/string-format/after.ts new file mode 100644 index 0000000..582ebe4 --- /dev/null +++ b/packages/core/src/packs/zod/fixtures/string-format/after.ts @@ -0,0 +1,10 @@ +import { z } from 'zod'; + +// Formats move to top-level factories; options and the rest of the chain stay. +export const contact = z.email({ error: 'Invalid address' }).optional(); // @uptide string-format at:.email( path:ZodString#email +export const id = z.uuid(); // @uptide string-format at:.uuid( path:ZodString#uuid +export const createdAt = z.iso.datetime({ offset: true }); // @uptide string-format at:.datetime( path:ZodString#datetime + +// Left for a person: a refinement before the format, and options on z.string() itself. +export const trimmed = z.string().trim().email(); // @uptide string-format keep at:.email( path:ZodString#email +export const described = z.string({ description: 'link' }).url(); // @uptide string-format keep at:.url( path:ZodString#url diff --git a/packages/core/src/packs/zod/fixtures/string-format/before.ts b/packages/core/src/packs/zod/fixtures/string-format/before.ts new file mode 100644 index 0000000..f5ba292 --- /dev/null +++ b/packages/core/src/packs/zod/fixtures/string-format/before.ts @@ -0,0 +1,10 @@ +import { z } from 'zod'; + +// Formats move to top-level factories; options and the rest of the chain stay. +export const contact = z.string().email({ message: 'Invalid address' }).optional(); // @uptide string-format at:.email( path:ZodString#email +export const id = z.string().uuid(); // @uptide string-format at:.uuid( path:ZodString#uuid +export const createdAt = z.string().datetime({ offset: true }); // @uptide string-format at:.datetime( path:ZodString#datetime + +// Left for a person: a refinement before the format, and options on z.string() itself. +export const trimmed = z.string().trim().email(); // @uptide string-format keep at:.email( path:ZodString#email +export const described = z.string({ description: 'link' }).url(); // @uptide string-format keep at:.url( path:ZodString#url diff --git a/packages/core/src/packs/zod/ground-truth.json b/packages/core/src/packs/zod/ground-truth.json new file mode 100644 index 0000000..7d7179f --- /dev/null +++ b/packages/core/src/packs/zod/ground-truth.json @@ -0,0 +1,204 @@ +{ + "package": "zod", + "repos": [ + { + "fixture": "fixtures/repos/storefront", + "from": "3.25.76", + "to": "4.6.5", + "why": "synthetic pnpm workspace of this repository (fixtures/truth/storefront.json): error params, string formats, ip, generics across two workspaces", + "findings": [ + { + "file": "packages/api/src/queue/consumer.ts", + "line": 2, + "rule": "types" + }, + { + "file": "packages/api/src/queue/consumer.ts", + "line": 12, + "rule": "types" + }, + { + "file": "packages/api/src/queue/consumer.ts", + "line": 19, + "rule": "types" + }, + { + "file": "packages/api/src/routes/customers.ts", + "line": 5, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/customers.ts", + "line": 6, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/customers.ts", + "line": 7, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/customers.ts", + "line": 12, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/customers.ts", + "line": 13, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/orders.ts", + "line": 5, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/orders.ts", + "line": 6, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/orders.ts", + "line": 10, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/orders.ts", + "line": 13, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/orders.ts", + "line": 17, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/orders.ts", + "line": 18, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/webhooks.ts", + "line": 4, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/webhooks.ts", + "line": 6, + "rule": "string-format" + }, + { + "file": "packages/api/src/routes/webhooks.ts", + "line": 8, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/webhooks.ts", + "line": 9, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/webhooks.ts", + "line": 14, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/webhooks.ts", + "line": 15, + "rule": "error-params" + }, + { + "file": "packages/api/src/routes/webhooks.ts", + "line": 16, + "rule": "string-format" + }, + { + "file": "packages/shared/src/address.ts", + "line": 4, + "rule": "error-params" + }, + { + "file": "packages/shared/src/address.ts", + "line": 5, + "rule": "error-params" + }, + { + "file": "packages/shared/src/address.ts", + "line": 7, + "rule": "error-params" + }, + { + "file": "packages/shared/src/address.ts", + "line": 10, + "rule": "error-params" + }, + { + "file": "packages/shared/src/errors.ts", + "line": 8, + "rule": "generic" + }, + { + "file": "packages/shared/src/errors.ts", + "line": 25, + "rule": "generic" + }, + { + "file": "packages/shared/src/monitoring.ts", + "line": 5, + "rule": "string-format" + }, + { + "file": "packages/shared/src/monitoring.ts", + "line": 6, + "rule": "ip" + }, + { + "file": "packages/shared/src/monitoring.ts", + "line": 9, + "rule": "string-format" + }, + { + "file": "packages/shared/src/product.ts", + "line": 4, + "rule": "string-format" + }, + { + "file": "packages/shared/src/product.ts", + "line": 5, + "rule": "error-params" + }, + { + "file": "packages/shared/src/product.ts", + "line": 6, + "rule": "error-params" + }, + { + "file": "packages/shared/src/product.ts", + "line": 8, + "rule": "error-params" + }, + { + "file": "packages/shared/src/product.ts", + "line": 11, + "rule": "string-format" + }, + { + "file": "packages/shared/src/product.ts", + "line": 12, + "rule": "string-format" + }, + { + "file": "packages/shared/src/product.ts", + "line": 16, + "rule": "string-format" + }, + { + "file": "packages/shared/src/product.ts", + "line": 17, + "rule": "error-params" + } + ] + } + ], + "$comment": "Ground truth for the zod pack. Public repositories at the commit before they upgraded zod 3 to 4, each with every site the real upgrade changed for zod or the compiler rejected (docs/packs.md, \"Ground truth\"). The storefront fixture is synthetic: scored, never counted toward verified; its breaking sites are the tsc errors of fixtures/truth/storefront.json, its deprecated ones the zod 3 APIs zod 4 deprecates (string formats, ZodIssue, flatten)." +} diff --git a/packages/core/src/packs/zod/index.ts b/packages/core/src/packs/zod/index.ts index 157590a..14d9e35 100644 --- a/packages/core/src/packs/zod/index.ts +++ b/packages/core/src/packs/zod/index.ts @@ -12,7 +12,8 @@ import { import type { Finding } from '../../domain/report.js'; import { satisfies } from '../../fetch/range.js'; import { onReset } from '../../shared-state.js'; -import type { MigrationPack, PackContext, TransformResult } from '../types.js'; +import type { Pack, PackMeta } from '../contract.js'; +import type { PackContext, TransformResult } from '../types.js'; import { zodGuide } from './guide.js'; import { defaultMessageSites } from './messages.js'; @@ -240,14 +241,12 @@ function parse(text: string): SourceFile { } return source; } -function transform(text: string, finding: Finding, context: PackContext): TransformResult { - if (!zodPack.supports(context.from, context.to)) - return { text, applied: false, reason: 'pack supports zod 3 to 4 only' }; - const rule = zodPack.rules.find( - (r) => r.kinds.includes(finding.change.kind) && r.symbols.test(finding.change.path), - ); - if (!rule || !['error-params', 'string-format'].includes(rule.id)) - return { text, applied: false, reason: 'no mechanical rule for this change kind and symbol' }; +/** Runs `each` on the calls around the reported site, innermost first, until one answers. */ +function atSite( + text: string, + finding: Finding, + each: (call: CallExpression) => TransformResult | undefined, +): TransformResult { const source = parse(text); const lines = text.split('\n'); const offset = @@ -259,43 +258,86 @@ function transform(text: string, finding: Finding, context: PackContext): Transf .filter((c) => c.getStart() <= offset && c.getEnd() >= offset) .sort((a, b) => a.getWidth() - b.getWidth()); for (const call of calls) { - const result = - finding.change.kind === 'deprecated' - ? context.includeDeprecated - ? formatTransform(text, call) - : undefined - : errorTransform(text, call); + const result = each(call); if (result) return result; } return { text, applied: false, reason: 'no safe mechanical transform for this reported site' }; } -export const zodPack: MigrationPack = { +function transform(text: string, finding: Finding, context: PackContext): TransformResult { + if (!zodPack.supports(context.from, context.to)) + return { text, applied: false, reason: 'pack supports zod 3 to 4 only' }; + const rule = zodPack.rules.find( + (r) => r.kinds.includes(finding.change.kind) && r.symbols.test(finding.change.path), + ); + if (!rule?.rewrite) + return { text, applied: false, reason: 'no mechanical rule for this change kind and symbol' }; + return rule.rewrite(text, finding, context); +} +const meta: PackMeta = { + package: 'zod', + from: '>=3 <4', + to: '>=4 <5', + sources: [ + { title: 'Zod 4 migration guide', url: 'https://zod.dev/v4/changelog' }, + { title: 'Zod 4 release notes', url: 'https://zod.dev/v4' }, + ], + maintainer: 'uptide-dev', +}; +export const zodPack: Pack = { name: 'zod', + meta, defaultTarget: '4.6.5', - supports: (from, to) => satisfies(from, '>=3 <4') && satisfies(to, '>=4 <5'), + supports: (from, to) => satisfies(from, meta.from) && satisfies(to, meta.to), rules: [ { id: 'error-params', + summary: 'required_error and invalid_type_error become one error callback', + severity: 'breaking', kinds: ['signature', 'type', 'required'], symbols: /string|number|boolean|bigint|date|symbol|undefined|null|void|any|unknown|never|enum|object|literal|array|tuple|union|record|map|set|TS(?:2353|2769|2345)/, guide: zodGuide.errors, + rewrite: (text, finding) => atSite(text, finding, (call) => errorTransform(text, call)), }, { id: 'string-format', + summary: 'z.string().email() and the other formats move to top-level factories', + severity: 'deprecated', kinds: ['deprecated'], symbols: /#(email|uuid|url|base64|datetime)$/, guide: zodGuide.formats, + rewrite: (text, finding, context) => + atSite(text, finding, (call) => + context.includeDeprecated ? formatTransform(text, call) : undefined, + ), + }, + { + id: 'ip', + summary: 'z.string().ip() was removed: z.ipv4(), z.ipv6() or their union', + severity: 'breaking', + kinds: ['removed'], + symbols: /#ip$/, + guide: zodGuide.ip, }, - { id: 'ip', kinds: ['removed'], symbols: /#ip$/, guide: zodGuide.ip }, { id: 'types', + summary: 'ZodType generics and ZodTypeDef changed; type-level code is migrated by the agent', + severity: 'breaking', kinds: ['type', 'signature', 'removed', 'narrowed', 'cause'], symbols: /./, guide: zodGuide.generics, perFile: true, }, ], + behavior: [ + { + id: 'default-messages', + summary: + 'Zod 4 words its default error messages differently; code and tests matching the old text keep compiling', + reported: ['decision', 'test-follow-up'], + }, + ], + instructions: zodGuide.generics, transform, guide: (f) => zodPack.rules.find((r) => r.kinds.includes(f.change.kind) && r.symbols.test(f.change.path)) diff --git a/packages/core/src/packs/zod/verification.json b/packages/core/src/packs/zod/verification.json new file mode 100644 index 0000000..d596ce8 --- /dev/null +++ b/packages/core/src/packs/zod/verification.json @@ -0,0 +1,6 @@ +{ + "status": "verified", + "truth": "", + "repos": 2, + "breaking": { "precision": 1, "recall": 1, "falsePositives": 0 } +} diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index bb07ae1..0302380 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -5,5 +5,6 @@ "outDir": "dist", "types": ["node"] }, - "include": ["src"] + "include": ["src"], + "exclude": ["src/packs/*/fixtures"] } From 03ef88c8f6b2c7e182e3d07297583a584e6a1bf5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 06:47:08 +0000 Subject: [PATCH 2/9] feat(cli): uptide pack new and uptide pack test pack new scaffolds a pack that lints and passes its tests as written and registers it. pack test runs its fixtures, then check with the pack on each ground-truth repository (fetched shallow at the pinned commit, installed with scripts off, cached), and prints precision and recall per rule, overall and for breaking findings, with every false positive and false negative. --json for CI. pnpm uptide runs the CLI from source. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01B2T8GmEVnZwfNrugajEMqi Signed-off-by: Claude --- package.json | 5 +- packages/cli/src/cli.ts | 105 +++++- packages/cli/src/pack.ts | 160 +++++++++ packages/core/package.json | 1 + packages/core/src/errors.ts | 8 +- packages/core/src/index.ts | 36 ++ packages/core/src/packs/tooling/fixtures.ts | 242 ++++++++++++++ packages/core/src/packs/tooling/pack-test.ts | 308 ++++++++++++++++++ .../core/src/packs/tooling/scaffold.test.ts | 144 ++++++++ packages/core/src/packs/tooling/scaffold.ts | 166 ++++++++++ packages/core/src/packs/tooling/truth.ts | 204 ++++++++++++ scripts/pack-test-report.ts | 18 + scripts/packs-fetch.ts | 49 +++ 13 files changed, 1443 insertions(+), 3 deletions(-) create mode 100644 packages/cli/src/pack.ts create mode 100644 packages/core/src/packs/tooling/fixtures.ts create mode 100644 packages/core/src/packs/tooling/pack-test.ts create mode 100644 packages/core/src/packs/tooling/scaffold.test.ts create mode 100644 packages/core/src/packs/tooling/scaffold.ts create mode 100644 packages/core/src/packs/tooling/truth.ts create mode 100644 scripts/pack-test-report.ts create mode 100644 scripts/packs-fetch.ts diff --git a/package.json b/package.json index 71fe703..c2d6931 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,10 @@ "build:stripe-changelog": "tsx scripts/build-stripe-changelog.ts", "eval:stripe-fixture": "tsx scripts/eval-stripe-fixture.ts", "eval:action": "tsx scripts/eval-action.ts", - "public:check": "node scripts/public-tree.mjs . --require-denylist" + "public:check": "node scripts/public-tree.mjs . --require-denylist", + "uptide": "node --conditions=source --import tsx packages/cli/src/index.ts", + "packs:fetch": "node --conditions=source --import tsx scripts/packs-fetch.ts", + "packs:queue": "node --conditions=source --import tsx scripts/packs-queue.ts" }, "devDependencies": { "@biomejs/biome": "^2.5.14", diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index cccbf4e..4f20aa8 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -1,7 +1,17 @@ import { readFileSync } from 'node:fs'; import { basename, resolve } from 'node:path'; import type { CheckReport, CheckResult, FixReport } from '@uptide/core'; -import { formatFix, isFailure, PRICE_DATE, priceFor, selectLlm, TIER_LEGEND } from '@uptide/core'; +import { + formatFix, + isFailure, + PRICE_DATE, + priceFor, + registeredPacks, + scaffoldPack, + selectLlm, + TIER_LEGEND, + testPack, +} from '@uptide/core'; import { Command, CommanderError } from 'commander'; import { describeRepo, detectRepo, type Repo } from './detect.js'; import { defaultEngine, type Engine } from './engine.js'; @@ -29,6 +39,7 @@ import { writePlanHtml } from './html/plan.js'; import { openHtml, writeHtml } from './html/write.js'; import { INVOCATION } from './invocation.js'; import { type Io, type Ui, type UiFlags, uiOf } from './io.js'; +import { formatPackTest, packRoot, packsToTest } from './pack.js'; import { PRIVACY } from './privacy.js'; import { createProgress, elapsed, type Progress } from './progress.js'; import { @@ -1166,6 +1177,98 @@ ${EXIT_CODES('description rendered (and updated unless --preview)', 'not used')} }), ); + const packs = program + .command('pack') + .description('Write and score migration packs (inside an uptide checkout; docs/packs.md)'); + packs + .command('new') + .description('Scaffold a pack: one example rule, a fixture pair, a test, empty ground truth') + .argument('', 'the npm package the pack migrates') + .requiredOption( + '--from ', + 'installed versions it migrates from, a semver range (">=6 <7")', + ) + .requiredOption('--to ', 'target versions, a semver range (">=7 <8")') + .option('--maintainer ', 'who answers for the pack: a GitHub handle') + .option('--cwd ', 'a directory inside the uptide checkout') + .option('--json', 'machine-readable output') + .addHelpText( + 'after', + ` +Example: + $ uptide pack new ai --from ">=6 <7" --to ">=7 <8" --maintainer @octocat +${EXIT_CODES('scaffolded', 'not used')}`, + ) + .action((name: string, flags: Shared & { from: string; to: string; maintainer?: string }) => + act(flags, async ({ cwd }) => { + const root = packRoot(cwd); + const result = scaffoldPack({ + root, + package: name, + from: flags.from, + to: flags.to, + ...(flags.maintainer ? { maintainer: flags.maintainer } : {}), + }); + if (flags.json) emit(result); + else { + io.out(`Scaffolded the ${name} pack (${result.constant}), registered as a candidate:\n`); + for (const file of result.files) io.out(` ${file}\n`); + io.out( + `\nNext: replace the example rule, add fixtures and ground truth (docs/packs.md), then\n pnpm uptide pack test ${name}\n`, + ); + } + return EXIT.ok; + }), + ); + packs + .command('test') + .description( + 'Score packs against their fixtures and ground truth: precision and recall per rule', + ) + .argument('[package]', 'one pack; every registered pack when omitted') + .option('--offline', 'never fetch: use only ground-truth repositories already cached') + .option('--fixtures-only', 'run the fixtures, skip the ground-truth repositories') + .option('--write', "record this run's result in the pack's verification.json") + .option('--cwd ', 'a directory inside the uptide checkout') + .option('--json', 'machine-readable output for CI') + .option('--no-color', 'no color (NO_COLOR is respected too)') + .addHelpText( + 'after', + ` +Ground-truth repositories are fetched once (shallow, the pinned commit only) and installed with +lifecycle scripts off into ~/.cache/uptide/ground-truth (UPTIDE_GROUND_TRUTH_CACHE overrides it). + +Examples: + $ uptide pack test zod + $ uptide pack test --json > pack-test.json + $ uptide pack test ai --write record the status verification.json claims +${EXIT_CODES('every pack passed', 'a false positive among breaking findings, a failing fixture, or a stale verification.json')}`, + ) + .action( + ( + name: string | undefined, + flags: Shared & { offline?: boolean; fixturesOnly?: boolean; write?: boolean }, + ) => + act(flags, async ({ ui, cwd }) => { + const root = packRoot(cwd); + const reports = []; + for (const entry of packsToTest(root, registeredPacks(), name)) { + const report = await testPack(entry, { + root, + ...(flags.offline ? { offline: true } : {}), + ...(flags.fixturesOnly ? { fixturesOnly: true } : {}), + ...(flags.write ? { write: true } : {}), + log: (line) => io.err(`${line}\n`), + }); + reports.push(report); + if (!flags.json) io.out(formatPackTest(report, ui.color)); + } + const passed = reports.every((r) => r.passed); + if (flags.json) emit({ passed, packs: reports }); + return passed ? EXIT.ok : EXIT.breaking; + }), + ); + program .command('diff') .description('Diff the public API of an npm package between two versions') diff --git a/packages/cli/src/pack.ts b/packages/cli/src/pack.ts new file mode 100644 index 0000000..8f822fb --- /dev/null +++ b/packages/cli/src/pack.ts @@ -0,0 +1,160 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import type { PackTestReport, RegisteredPack, Tally } from '@uptide/core'; +import pc from 'picocolors'; +import { CliError } from './errors.js'; + +/** The uptide checkout `cwd` is in: packs are contributor work, done in the source tree. */ +export function packRoot(cwd: string): string { + let dir = cwd; + for (;;) { + if (existsSync(join(dir, 'packages', 'core', 'src', 'packs', 'registry.ts'))) return dir; + const parent = dirname(dir); + if (parent === dir) + throw new CliError('uptide pack works inside an uptide checkout', { + why: 'packs live in packages/core/src/packs of https://github.com/uptide-dev/uptide', + next: 'git clone https://github.com/uptide-dev/uptide && cd uptide && pnpm install', + }); + dir = parent; + } +} + +/** + * The packs to test: what this build carries, checked against the checkout's registry. A pack + * added or edited since the build would be tested as it was, so a missing one is an error. + */ +export function packsToTest( + root: string, + registered: readonly RegisteredPack[], + name: string | undefined, +): RegisteredPack[] { + const onDisk = [ + ...readFileSync(join(root, 'packages', 'core', 'src', 'packs', 'registry.ts'), 'utf8').matchAll( + /\{ dir: '([^']+)'/g, + ), + ].map((m) => m[1] as string); + const missing = onDisk.filter((dir) => !registered.some((e) => e.dir === dir)); + if (missing.length > 0) + throw new CliError(`this build of uptide does not include the pack in ${missing.join(', ')}`, { + why: 'pack test runs the packs compiled into the CLI; `pnpm uptide` runs them from source', + next: `pnpm uptide pack test${name ? ` ${name}` : ''}`, + }); + if (!name) return [...registered]; + const entry = registered.find((e) => e.pack.name === name || e.dir === name); + if (!entry) + throw new CliError(`no pack for ${name}`, { + next: `uptide pack new ${name} --from --to `, + }); + return [entry]; +} + +const pct = (n: number): string => `${Math.round(n * 100)}%`; + +function row(name: string, t: Tally): string { + return ` ${name.padEnd(28)} ${pct(t.precision).padStart(9)} ${pct(t.recall).padStart(7)} ${String(t.truePositives).padStart(5)} ${String(t.falsePositives).padStart(4)} ${String(t.falseNegatives).padStart(4)}`; +} + +/** What `uptide pack test` prints for one pack. */ +export function formatPackTest(report: PackTestReport, color = false): string { + const c = pc.createColors(color); + const out: string[] = []; + const recorded = report.recorded === report.status ? '' : `, recorded ${report.recorded}`; + out.push(c.bold(`uptide pack test ${report.package} · ${report.status}${recorded}`)); + for (const problem of report.problems) out.push(` ${c.red('✗')} ${problem}`); + + const f = report.fixtures; + const fixtureSites = Object.values(f.rules).reduce( + (n, t) => n + t.truePositives + t.falseNegatives, + 0, + ); + const fixturesOk = + f.rewriteFailures.length + f.falsePositives.length + f.falseNegatives.length === 0 && + f.unknown.length === 0; + out.push( + `fixtures ${f.cases.length} case${f.cases.length === 1 ? '' : 's'}, ${fixtureSites} site${fixtureSites === 1 ? '' : 's'} · ${fixturesOk ? c.green('pass') : c.red('fail')}`, + ); + for (const u of f.unknown) + out.push(` ${c.red('✗')} ${u.file}:${u.line} names no rule: ${u.rule}`); + for (const r of f.rewriteFailures) + out.push(` ${c.red('✗')} ${r.file}${r.line ? `:${r.line}` : ''} ${r.message}`); + for (const s of f.falsePositives) + out.push(` ${c.red('✗')} ${s.file}:${s.line} ${s.rule} detected, not marked`); + for (const s of f.falseNegatives) + out.push(` ${c.red('✗')} ${s.file}:${s.line} ${s.rule} marked, not detected`); + + if (report.repos.length === 0) + out.push('ground truth no repository yet: the pack ships as a candidate'); + else { + out.push('ground truth'); + for (const r of report.repos) { + const head = ` ${r.label.padEnd(44)} ${report.package} ${r.from} → ${r.to}`; + if (r.error) { + out.push(`${head} ${c.red(`error: ${r.error}`)}`); + continue; + } + out.push( + `${head} predicted ${r.predicted.length}, expected ${r.expected.length}, false positives ${r.falsePositives.length}, false negatives ${r.falseNegatives.length}`, + ); + if (r.installedMismatch) out.push(` ${c.yellow('!')} ${r.installedMismatch}`); + } + out.push(''); + out.push( + ` ${'rule'.padEnd(28)} ${'precision'.padStart(9)} ${'recall'.padStart(7)} ${'tp'.padStart(5)} ${'fp'.padStart(4)} ${'fn'.padStart(4)}`, + ); + for (const [rule, t] of Object.entries(report.rules).sort(([a], [b]) => a.localeCompare(b))) + out.push(row(rule, t)); + out.push(row('overall (sites)', report.overall)); + out.push(c.bold(row('breaking (sites)', report.breaking))); + const listed = ( + title: string, + sites: { label: string; file: string; line: number; text: string }[], + ) => { + if (sites.length === 0) return; + out.push(''); + out.push(`${title} (${sites.length})`); + for (const s of sites) out.push(` ${s.label} ${s.file}:${s.line} ${s.text}`); + }; + listed( + 'false positives', + report.repos.flatMap((r) => + r.falsePositives.map((s) => ({ + label: r.label, + file: s.file, + line: s.line, + text: `${s.rule} (${s.severity ?? 'breaking'})`, + })), + ), + ); + listed( + 'false negatives', + report.repos.flatMap((r) => + r.falseNegatives.map((s) => ({ label: r.label, file: s.file, line: s.line, text: s.rule })), + ), + ); + listed( + 'reported under another rule', + report.repos.flatMap((r) => + r.wrongRule.map((s) => ({ + label: r.label, + file: s.file, + line: s.line, + text: `${s.rule}, expected ${s.expected}`, + })), + ), + ); + } + out.push(''); + if (report.stale) + out.push( + `${c.red('✗')} verification.json does not match this run: \`uptide pack test ${report.package} --write\` records it`, + ); + const breakingFp = report.breaking.falsePositives; + out.push( + report.passed + ? c.green(`PASS ${report.package}: ${report.status}`) + : c.red( + `FAIL ${report.package}${breakingFp ? `: ${breakingFp} false positive${breakingFp === 1 ? '' : 's'} among breaking findings` : ''}`, + ), + ); + return `${out.join('\n')}\n`; +} diff --git a/packages/core/package.json b/packages/core/package.json index 9725f7d..8b9d972 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -8,6 +8,7 @@ "types": "./dist/index.d.ts", "exports": { ".": { + "source": "./src/index.ts", "types": "./dist/index.d.ts", "import": "./dist/index.js" }, diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.ts index e157db8..fe2c3fe 100644 --- a/packages/core/src/errors.ts +++ b/packages/core/src/errors.ts @@ -103,7 +103,13 @@ export type ErrorCode = | 'ANALYSIS_STACK_OVERFLOW' | 'NO_FIXER' | 'RUN_STALE' - | 'INVALID_WORKSPACE'; + | 'INVALID_WORKSPACE' + | 'GROUND_TRUTH_NOT_CACHED' + | 'GROUND_TRUTH_FETCH' + | 'GROUND_TRUTH_INSTALL' + | 'INVALID_GROUND_TRUTH' + | 'NOT_UPTIDE_CHECKOUT' + | 'PACK_EXISTS'; /** Stable across rendering and worker serialization. Human wording is not an API. */ export class UptideError extends Error { override readonly name = 'UptideError'; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 872d944..f24abf1 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -143,7 +143,43 @@ export { ACCEPTED_KEYS, KEY_ENV, selectLlm } from './llm/config.js'; export { DEFAULT_MAX_COST_USD, providerFixer } from './llm/fixer.js'; export { DEFAULT_MODELS, PRICE_DATE, PRICE_SOURCES, PRICES, priceFor } from './llm/pricing.js'; export type { Provider } from './llm/types.js'; +export { + type BehaviorNote, + definePack, + type GroundTruth, + type GroundTruthFinding, + type GroundTruthRepo, + type Pack, + type PackMeta, + type PackRule, + type PackStatus, + type PackVerification, + type RegisteredPack, + replaceAtSite, + VERIFIED_MIN_REPOS, +} from './packs/contract.js'; +export { activePacks, packStatus, registeredPacks } from './packs/index.js'; export { sdkApiVersion, stripeConcerns, stripePack } from './packs/stripe/index.js'; +export { type FixtureResult, runFixtures } from './packs/tooling/fixtures.js'; +export { + GENERIC_RULE, + type PackTestOptions, + type PackTestReport, + type RepoScore, + type Tally, + testPack, +} from './packs/tooling/pack-test.js'; +export { + type ScaffoldOptions, + type ScaffoldResult, + scaffoldPack, +} from './packs/tooling/scaffold.js'; +export { + cachedRepoDir, + ensureRepo, + type PinnedRepo, + truthCacheDir, +} from './packs/tooling/truth.js'; export type { MigrationPack, MigrationRule } from './packs/types.js'; export { zodPack } from './packs/zod/index.js'; export { type PlanOptions, type PlanServices, upgradePlan } from './plan/gather.js'; diff --git a/packages/core/src/packs/tooling/fixtures.ts b/packages/core/src/packs/tooling/fixtures.ts new file mode 100644 index 0000000..0cb3f2a --- /dev/null +++ b/packages/core/src/packs/tooling/fixtures.ts @@ -0,0 +1,242 @@ +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import type { ChangeKind } from '../../domain/change.js'; +import type { Finding } from '../../domain/report.js'; +import { minVersion } from '../../fetch/range.js'; +import { detectIn, type Pack, ruleFor } from '../contract.js'; + +/** + * Fixtures: `fixtures//before.ts` and `after.ts` in the pack's directory. A site is a + * line ending in a marker comment: + * + * z.string({ required_error: 'A' }); // @uptide error-params at:z.string + * z.string({ errorMap: m }); // @uptide error-params keep at:z.string + * + * `keep` marks a site the rule must leave alone. `at:` is where on the line the site + * starts (default: the first non-blank column), `kind:` and `path:` the change `check` would + * report there (default: the rule's first kind, and the rule id). Every marked site of a rule + * with `rewrite` is rewritten, bottom-up, and the result must equal `after.ts` byte for byte + * (markers included: they are comments). A rule or note with `detect` must find exactly its + * marked lines in `before.ts`, and nothing on any other line. + */ +export interface FixtureSite { + case: string; + file: string; + line: number; + rule: string; + keep: boolean; + at?: string; + kind?: ChangeKind; + path?: string; +} + +export interface FixtureResult { + cases: string[]; + /** Per rule: detection against the markers (rules with `detect`), rewrites against `after.ts`. */ + rules: Record; + falsePositives: { rule: string; file: string; line: number }[]; + falseNegatives: { rule: string; file: string; line: number }[]; + /** A case whose rewritten `before.ts` differs from `after.ts`, or a rewrite that should not have applied. */ + rewriteFailures: { case: string; file: string; line?: number; message: string }[]; + /** A marker naming no rule or note of the pack. */ + unknown: { file: string; line: number; rule: string }[]; +} + +const MARKER = /\/\/\s*@uptide\s+([\w-]+)((?:\s+[^\s]+)*)\s*$/; + +export function parseMarkers(text: string, caseName: string, file: string): FixtureSite[] { + const sites: FixtureSite[] = []; + text.split('\n').forEach((line, i) => { + const match = MARKER.exec(line); + if (!match) return; + const site: FixtureSite = { + case: caseName, + file, + line: i + 1, + rule: match[1] as string, + keep: false, + }; + for (const word of (match[2] ?? '').trim().split(/\s+/).filter(Boolean)) { + if (word === 'keep') site.keep = true; + else if (word.startsWith('at:')) site.at = word.slice(3); + else if (word.startsWith('kind:')) site.kind = word.slice(5) as ChangeKind; + else if (word.startsWith('path:')) site.path = word.slice(5); + } + sites.push(site); + }); + return sites; +} + +/** Versions inside the pack's ranges, for the context a fixture runs under. */ +export function fixtureVersions(pack: Pick): { from: string; to: string } { + const from = minVersion(pack.meta.from) ?? '0.0.0'; + const to = minVersion(pack.meta.to) ?? '0.0.0'; + return { from, to }; +} + +/** The finding `check` would report at a marked site. */ +export function fixtureFinding( + pack: Pick, + site: FixtureSite, + text: string, +): Finding { + const { from, to } = fixtureVersions(pack); + const rule = pack.rules.find((r) => r.id === site.rule); + const lineText = text.split('\n')[site.line - 1] ?? ''; + const at = site.at ? lineText.indexOf(site.at) : -1; + const column = (at >= 0 ? at : lineText.search(/\S|$/)) + 1; + const kind = site.kind ?? rule?.kinds[0] ?? 'signature'; + const path = site.path ?? site.at ?? site.rule; + const severity = rule?.severity ?? 'breaking'; + return { + change: { + package: pack.meta.package, + from, + to, + path, + kind, + severity, + source: 'types', + confidence: 1, + }, + usage: { + file: site.file, + line: site.line, + column, + endLine: site.line, + endColumn: column, + symbolPath: path, + access: 'call', + snippet: lineText.trim(), + via: 'direct', + }, + severity, + confidence: 1, + fixability: 'mechanical', + reason: '', + rule: site.rule, + }; +} + +export function fixtureCases(dir: string): string[] { + const root = join(dir, 'fixtures'); + if (!existsSync(root)) return []; + return readdirSync(root, { withFileTypes: true }) + .filter((e) => e.isDirectory() && existsSync(join(root, e.name, 'before.ts'))) + .map((e) => e.name) + .sort(); +} + +/** Every fixture case of the pack in `dir`, scored the way ground truth is. */ +export function runFixtures(pack: Pack, dir: string): FixtureResult { + const result: FixtureResult = { + cases: [], + rules: {}, + falsePositives: [], + falseNegatives: [], + rewriteFailures: [], + unknown: [], + }; + const tally = (rule: string) => { + result.rules[rule] ??= { truePositives: 0, falsePositives: 0, falseNegatives: 0 }; + return result.rules[rule]; + }; + for (const rule of pack.rules) tally(rule.id); + for (const note of pack.behavior.filter((b) => b.detect)) tally(note.id); + const known = new Set([...pack.rules.map((r) => r.id), ...pack.behavior.map((b) => b.id)]); + const detecting = new Set([ + ...pack.rules.filter((r) => r.detect).map((r) => r.id), + ...pack.behavior.filter((b) => b.detect && b.reported.includes('finding')).map((b) => b.id), + ]); + for (const name of fixtureCases(dir)) { + result.cases.push(name); + const file = `fixtures/${name}/before.ts`; + const before = readFileSync(join(dir, file), 'utf8'); + const afterPath = join(dir, 'fixtures', name, 'after.ts'); + const after = existsSync(afterPath) ? readFileSync(afterPath, 'utf8') : before; + const sites = parseMarkers(before, name, file); + for (const site of sites) + if (!known.has(site.rule)) + result.unknown.push({ file: site.file, line: site.line, rule: site.rule }); + + // Detection: exactly the marked lines, for every rule or note that detects. + const found = detectIn(pack, file, before); + for (const id of detecting) { + const expected = new Set(sites.filter((s) => s.rule === id && !s.keep).map((s) => s.line)); + const got = new Set(found.filter((f) => f.rule === id).map((f) => f.line)); + for (const line of got) { + if (expected.has(line)) tally(id).truePositives++; + else { + tally(id).falsePositives++; + result.falsePositives.push({ rule: id, file, line }); + } + } + for (const line of expected) + if (!got.has(line)) { + tally(id).falseNegatives++; + result.falseNegatives.push({ rule: id, file, line }); + } + } + + // Rewrites: every marked site of a rule that rewrites, bottom-up so lines stay put. + let text = before; + const { from, to } = fixtureVersions(pack); + const context = { from, to, includeDeprecated: true }; + for (const site of [...sites].sort((a, b) => b.line - a.line)) { + const rule = pack.rules.find((r) => r.id === site.rule); + if (!rule?.rewrite) continue; + const finding = fixtureFinding(pack, site, text); + if (ruleFor(pack.rules, finding)?.id !== rule.id) continue; + const out = pack.transform(text, finding, context); + if (site.keep) { + if (out.applied) { + tally(rule.id).falsePositives++; + result.rewriteFailures.push({ + case: name, + file, + line: site.line, + message: `${rule.id} rewrote a site marked keep (${out.reason})`, + }); + } + continue; + } + if (!out.applied) { + tally(rule.id).falseNegatives++; + result.rewriteFailures.push({ + case: name, + file, + line: site.line, + message: `${rule.id} did not rewrite the site: ${out.reason}`, + }); + continue; + } + if (!rule.detect) tally(rule.id).truePositives++; + text = out.text; + } + if (text !== after) + result.rewriteFailures.push({ + case: name, + file: `fixtures/${name}/after.ts`, + message: `the rewritten before.ts differs from after.ts:\n${firstDifference(text, after)}`, + }); + } + return result; +} + +function firstDifference(got: string, want: string): string { + const a = got.split('\n'); + const b = want.split('\n'); + for (let i = 0; i < Math.max(a.length, b.length); i++) + if (a[i] !== b[i]) + return ` line ${i + 1}\n - expected: ${b[i] ?? '(end of file)'}\n + got: ${a[i] ?? '(end of file)'}`; + return ''; +} + +export function fixturesPass(result: FixtureResult): boolean { + return ( + result.falsePositives.length === 0 && + result.falseNegatives.length === 0 && + result.rewriteFailures.length === 0 && + result.unknown.length === 0 + ); +} diff --git a/packages/core/src/packs/tooling/pack-test.ts b/packages/core/src/packs/tooling/pack-test.ts new file mode 100644 index 0000000..910fc96 --- /dev/null +++ b/packages/core/src/packs/tooling/pack-test.ts @@ -0,0 +1,308 @@ +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { createFsSurfaceCache } from '../../cache/fs-surface-cache.js'; +import { check } from '../../check/check.js'; +import type { CheckReport } from '../../domain/report.js'; +import { + type GroundTruth, + type GroundTruthRepo, + type Pack, + type PackStatus, + type PackVerification, + publicRepos, + type RegisteredPack, + recordedStatus, + statusOf, + truthDigest, +} from '../contract.js'; +import { type FixtureResult, fixturesPass, runFixtures } from './fixtures.js'; +import { type EnsureOptions, ensureFixture, ensureRepo, label, truthProblems } from './truth.js'; + +/** A site as scored: repository-relative file, line, the rule it belongs to, its severity. */ +export interface ScoredSite { + file: string; + line: number; + rule: string; + severity?: 'breaking' | 'deprecated' | 'unverified'; +} + +export interface Tally { + truePositives: number; + falsePositives: number; + falseNegatives: number; + precision: number; + recall: number; +} + +export interface RepoScore { + label: string; + repo?: string; + commit?: string; + fixture?: string; + from: string; + to: string; + /** Set when the repository could not be scored: nothing else below is meaningful. */ + error?: string; + /** What the lockfile says is installed, when it is not the ground truth's `from`. */ + installedMismatch?: string; + predicted: ScoredSite[]; + expected: ScoredSite[]; + /** Predicted sites no expected finding has, at the line level. */ + falsePositives: ScoredSite[]; + /** Expected findings nothing was predicted at. */ + falseNegatives: ScoredSite[]; + /** Real sites reported under another rule: `rule` is predicted, `expected` the truth's. */ + wrongRule: (ScoredSite & { expected: string })[]; +} + +export interface PackTestReport { + package: string; + dir: string; + /** What the evidence below supports, and what `verification.json` says. */ + status: PackStatus; + recorded: PackStatus; + /** The recorded verification no longer matches this run (`--write` updates it). */ + stale: boolean; + verification: PackVerification; + problems: string[]; + fixtures: FixtureResult; + repos: RepoScore[]; + /** Ground truth, per rule: a site counts for a rule only when the rule matches too. */ + rules: Record; + /** Ground truth, at the line level, every severity. */ + overall: Tally; + /** Ground truth, at the line level, breaking findings only: the verified gate. */ + breaking: Tally; + passed: boolean; +} + +/** The rule id a scored site is filed under: a rule or note of the pack, else `generic`. */ +export const GENERIC_RULE = 'generic'; + +function ratio(a: number, b: number): number { + return b === 0 ? 1 : a / b; +} + +function tally(tp: number, fp: number, fn: number): Tally { + return { + truePositives: tp, + falsePositives: fp, + falseNegatives: fn, + precision: ratio(tp, tp + fp), + recall: ratio(tp, tp + fn), + }; +} + +const key = (s: { file: string; line: number }): string => `${s.file}:${s.line}`; + +/** + * The sites `check` shows for the package, from its plan: the findings worth acting on, + * grouped under the rule `fix` would apply. Repository-relative, as ground truth is written. + */ +export function predictedSites(report: CheckReport, pack: Pack): ScoredSite[] { + const own = new Set([...pack.rules.map((r) => r.id), ...pack.behavior.map((b) => b.id)]); + const sites = new Map(); + for (const p of report.packages) { + const names = p.members ? p.members.map((m) => m.name) : [p.name]; + if (!names.includes(pack.name)) continue; + const prefix = p.workspace === '.' || p.workspace === '*' ? '' : `${p.workspace}/`; + for (const group of p.plan ?? []) + for (const location of group.locations) { + const file = location.file.startsWith(prefix) ? location.file : `${prefix}${location.file}`; + const site: ScoredSite = { + file, + line: location.line, + rule: own.has(group.rule) ? group.rule : GENERIC_RULE, + severity: group.severity, + }; + // One site, one rule: the more severe group wins, as the plan lists it first. + if (!sites.has(key(site))) sites.set(key(site), site); + } + } + return [...sites.values()].sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line); +} + +/** One repository: what was predicted against what the ground truth expects. */ +export function scoreSites( + predicted: ScoredSite[], + expected: ScoredSite[], +): Pick { + const truth = new Map(expected.map((e) => [key(e), e])); + const seen = new Set(predicted.map(key)); + return { + falsePositives: predicted.filter((p) => !truth.has(key(p))), + falseNegatives: expected.filter((e) => !seen.has(key(e))), + wrongRule: predicted + .filter((p) => truth.has(key(p)) && truth.get(key(p))?.rule !== p.rule) + .map((p) => ({ ...p, expected: truth.get(key(p))?.rule as string })), + }; +} + +async function scoreRepo( + pack: Pack, + entry: GroundTruthRepo, + root: string, + options: EnsureOptions, +): Promise { + const expected = entry.findings.map((f) => ({ file: f.file, line: f.line, rule: f.rule })); + const base: RepoScore = { + label: label(entry), + ...(entry.repo ? { repo: entry.repo } : {}), + ...(entry.commit ? { commit: entry.commit } : {}), + ...(entry.fixture ? { fixture: entry.fixture } : {}), + from: entry.from, + to: entry.to, + predicted: [], + expected, + falsePositives: [], + falseNegatives: [], + wrongRule: [], + }; + try { + const dir = entry.fixture + ? await ensureFixture(root, entry.fixture, options) + : await ensureRepo({ repo: entry.repo as string, commit: entry.commit as string }, options); + options.log?.(`checking ${pack.name} ${entry.from} → ${entry.to} in ${label(entry)}`); + const report = await check({ + cwd: dir, + only: [pack.name], + targets: { [pack.name]: entry.to }, + packs: [pack], + // In this thread, with the pack under test: no worker loads the bundled registry. + cache: createFsSurfaceCache(), + }); + const installed = [ + ...new Set(report.packages.filter((p) => p.name === pack.name).map((p) => p.installed)), + ]; + if (installed.length > 0 && !installed.includes(entry.from)) + base.installedMismatch = `the lockfile has ${installed.join(', ')}, the ground truth says ${entry.from}`; + const predicted = predictedSites(report, pack); + return { ...base, predicted, ...scoreSites(predicted, expected) }; + } catch (err) { + return { ...base, error: (err as Error).message }; + } +} + +export interface PackTestOptions extends EnsureOptions { + /** The uptide checkout: packs live in `/packages/core/src/packs`. */ + root: string; + /** Fixtures only: no repository is fetched or checked. */ + fixturesOnly?: boolean; + /** Record this run's verification in the pack's `verification.json`. */ + write?: boolean; +} + +export function packsDir(root: string): string { + return join(root, 'packages', 'core', 'src', 'packs'); +} + +export function readTruth(dir: string): { truth: GroundTruth; text: string } | undefined { + const file = join(dir, 'ground-truth.json'); + if (!existsSync(file)) return undefined; + const text = readFileSync(file, 'utf8'); + return { truth: JSON.parse(text) as GroundTruth, text }; +} + +/** Fixtures and ground truth of one pack, scored; nothing is written. */ +export async function testPack( + entry: RegisteredPack, + options: PackTestOptions, +): Promise { + const { pack } = entry; + const dir = join(packsDir(options.root), entry.dir); + const problems: string[] = []; + if (!pack.meta.sources.length) problems.push('meta.sources lists no changelog or guide'); + if (!pack.meta.maintainer) problems.push('meta.maintainer is empty'); + const fixtures = runFixtures(pack, dir); + if (fixtures.cases.length === 0 && pack.rules.some((r) => r.rewrite || r.detect)) + problems.push('rules that rewrite or detect need fixtures (fixtures//before.ts)'); + const read = readTruth(dir); + if (!read) problems.push('no ground-truth.json'); + const truth = read?.truth ?? { package: pack.name, repos: [] }; + problems.push(...truthProblems(truth, pack.name)); + + const repos: RepoScore[] = []; + if (!options.fixturesOnly && problems.length === 0) + for (const repo of truth.repos) repos.push(await scoreRepo(pack, repo, options.root, options)); + + const scored = repos.filter((r) => !r.error); + const all = (k: K) => + scored.flatMap((r) => r[k]); + const fp = all('falsePositives'); + const fn = all('falseNegatives'); + const predicted = all('predicted'); + const overall = tally(predicted.length - fp.length, fp.length, fn.length); + const breakingPredicted = predicted.filter((p) => p.severity === 'breaking'); + const breakingFp = fp.filter((p) => p.severity === 'breaking'); + const breaking = tally( + breakingPredicted.length - breakingFp.length, + breakingFp.length, + fn.length, + ); + const rules: Record = {}; + const ids = new Set([ + ...pack.rules.map((r) => r.id), + ...pack.behavior.map((b) => b.id), + ...predicted.map((p) => p.rule), + ...all('expected').map((e) => e.rule), + ]); + for (const id of ids) { + let tp = 0; + let rfp = 0; + let rfn = 0; + for (const r of scored) { + const want = new Set(r.expected.filter((e) => e.rule === id).map(key)); + const got = new Set(r.predicted.filter((p) => p.rule === id).map(key)); + for (const k of got) (want.has(k) ? () => tp++ : () => rfp++)(); + for (const k of want) if (!got.has(k)) rfn++; + } + if (tp + rfp + rfn > 0) rules[id] = tally(tp, rfp, rfn); + } + + const status: PackStatus = + repos.some((r) => r.error) || options.fixturesOnly + ? 'candidate' + : statusOf(truth, { falsePositives: breakingFp.length, predicted: breakingPredicted.length }); + const verification: PackVerification = { + status, + truth: read ? truthDigest(read.text) : '', + repos: publicRepos(truth).length, + breaking: { + precision: Number(breaking.precision.toFixed(4)), + recall: Number(breaking.recall.toFixed(4)), + falsePositives: breakingFp.length, + }, + }; + const recorded = recordedStatus(entry.verification); + // Fixtures alone say nothing about ground truth: only a full run can find the record stale. + let stale = + !options.fixturesOnly && JSON.stringify(verification) !== JSON.stringify(entry.verification); + const scoredAll = problems.length === 0 && repos.every((r) => !r.error); + if (options.write && stale && scoredAll && !options.fixturesOnly) { + writeVerification(options.root, entry.dir, verification); + stale = false; + } + const passed = scoredAll && fixturesPass(fixtures) && breakingFp.length === 0 && !stale; + return { + package: pack.name, + dir: entry.dir, + status, + recorded, + stale, + verification, + problems, + fixtures, + repos, + rules, + overall, + breaking, + passed, + }; +} + +/** `--write`: the record the runtime gate reads, as this run measured it. */ +export function writeVerification(root: string, dir: string, record: PackVerification): string { + const file = join(packsDir(root), dir, 'verification.json'); + writeFileSync(file, `${JSON.stringify(record, null, 2)}\n`); + return file; +} diff --git a/packages/core/src/packs/tooling/scaffold.test.ts b/packages/core/src/packs/tooling/scaffold.test.ts new file mode 100644 index 0000000..42dbf64 --- /dev/null +++ b/packages/core/src/packs/tooling/scaffold.test.ts @@ -0,0 +1,144 @@ +import { spawnSync } from 'node:child_process'; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, expect, it } from 'vitest'; +import type { Pack, PackVerification } from '../contract.js'; +import { testPack } from './pack-test.js'; +import { parseRegistry, renderRegistry } from './registry-file.js'; +import { scaffoldPack } from './scaffold.js'; + +const repo = fileURLToPath(new URL('../../../../../', import.meta.url)); +const packs = fileURLToPath(new URL('..', import.meta.url)); +const root = mkdtempSync(join(tmpdir(), 'uptide-pack-new-')); +afterAll(() => rmSync(root, { recursive: true, force: true })); + +/** + * A checkout in miniature: the generated pack imports `../contract.js` and + * `../tooling/fixtures.js`, which forward to this tree's, and the checkout's biome and + * vitest are reachable as they are for a contributor. + */ +function miniCheckout(): string { + const home = join(root, 'packages', 'core', 'src', 'packs'); + mkdirSync(join(home, 'tooling'), { recursive: true }); + writeFileSync(join(home, 'registry.ts'), renderRegistry([])); + writeFileSync(join(home, 'contract.ts'), `export * from '${join(packs, 'contract.ts')}';\n`); + writeFileSync( + join(home, 'tooling', 'fixtures.ts'), + `export * from '${join(packs, 'tooling', 'fixtures.ts')}';\n`, + ); + symlinkSync(join(repo, 'node_modules'), join(root, 'node_modules')); + copyFileSync(join(repo, 'biome.json'), join(root, 'biome.json')); + copyFileSync(join(repo, '.gitignore'), join(root, '.gitignore')); + spawnSync('git', ['init', '-q'], { cwd: root }); + return home; +} + +it('pack new scaffolds a pack that lints, passes its own test and pack test; one added rule keeps it passing', async () => { + const home = miniCheckout(); + const result = scaffoldPack({ root, package: 'toy-lib', from: '>=1 <2', to: '>=2 <3' }); + expect(result).toMatchObject({ dir: 'toy-lib', constant: 'toyLibPack', formatted: true }); + expect(parseRegistry(readFileSync(join(home, 'registry.ts'), 'utf8'))).toEqual([ + { dir: 'toy-lib', name: 'toyLibPack' }, + ]); + expect(() => scaffoldPack({ root, package: 'toy-lib', from: '>=1 <2', to: '>=2 <3' })).toThrow( + /already exists/, + ); + + // Lint: the checkout's biome accepts every generated file as it is. + const biome = spawnSync( + join(repo, 'node_modules', '.bin', 'biome'), + ['check', '--no-errors-on-unmatched', 'packages'], + { cwd: root, encoding: 'utf8' }, + ); + expect(biome.stdout + biome.stderr).toContain('No fixes applied'); + expect(biome.status).toBe(0); + + // Add one rule: a note the compiler cannot see, detected in the source, with its fixture. + const index = join(home, 'toy-lib', 'index.ts'); + writeFileSync( + index, + readFileSync(index, 'utf8').replace( + ' behavior: [],', + ` behavior: [ + { + id: 'retry-default', + summary: 'connect() retries three times by default in 2.x; it never retried in 1.x', + reported: ['finding'], + detect: (text) => + text.split('\\n').flatMap((line, i) => { + const at = line.indexOf('connect('); + return at >= 0 && !line.includes('retries:') + ? [{ line: i + 1, column: at + 1, snippet: line.trim() }] + : []; + }), + }, + ],`, + ), + ); + mkdirSync(join(home, 'toy-lib', 'fixtures', 'retry-default'), { recursive: true }); + writeFileSync( + join(home, 'toy-lib', 'fixtures', 'retry-default', 'before.ts'), + [ + "import { connect } from 'toy-lib';", + '', + "export const a = connect('db'); // @uptide retry-default", + "export const b = connect('db', { retries: 0 }); // @uptide retry-default keep", + '', + ].join('\n'), + ); + + const pack = ((await import(index)) as { toyLibPack: Pack }).toyLibPack; + const verification = JSON.parse( + readFileSync(join(home, 'toy-lib', 'verification.json'), 'utf8'), + ) as PackVerification; + const report = await testPack({ dir: 'toy-lib', pack, verification }, { root }); + expect(report.problems).toEqual([]); + expect(report.fixtures.cases).toEqual(['example-rename', 'retry-default']); + expect(report.fixtures.rules['retry-default']).toEqual({ + truePositives: 1, + falsePositives: 0, + falseNegatives: 0, + }); + expect(report.fixtures.rewriteFailures).toEqual([]); + expect(report).toMatchObject({ status: 'candidate', stale: false, passed: true }); + + // The generated test, run by the checkout's vitest as `pnpm test` would. + const vitest = spawnSync( + join(repo, 'node_modules', '.bin', 'vitest'), + ['run', '--root', join(root, 'packages', 'core'), 'src/packs/toy-lib'], + { cwd: root, encoding: 'utf8' }, + ); + expect(vitest.stdout).toMatch(/1 passed/); + expect(vitest.status).toBe(0); +}, 120_000); + +it('a fixture marker naming no rule, or a rewrite that misses after.ts, fails pack test', async () => { + const { zodPack } = await import('../zod/index.js'); + const dir = mkdtempSync(join(tmpdir(), 'uptide-pack-fixture-')); + try { + mkdirSync(join(dir, 'fixtures', 'bad'), { recursive: true }); + writeFileSync( + join(dir, 'fixtures', 'bad', 'before.ts'), + "import { z } from 'zod';\nexport const s = z.string({ required_error: 'A' }); // @uptide error-params at:z.string\nexport const t = 1; // @uptide no-such-rule\n", + ); + writeFileSync(join(dir, 'fixtures', 'bad', 'after.ts'), 'not what the rule writes\n'); + const { runFixtures } = await import('./fixtures.js'); + const result = runFixtures(zodPack, dir); + expect(result.unknown).toEqual([ + { file: 'fixtures/bad/before.ts', line: 3, rule: 'no-such-rule' }, + ]); + expect(result.rewriteFailures[0]?.message).toMatch(/differs from after\.ts/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/core/src/packs/tooling/scaffold.ts b/packages/core/src/packs/tooling/scaffold.ts new file mode 100644 index 0000000..a7545ef --- /dev/null +++ b/packages/core/src/packs/tooling/scaffold.ts @@ -0,0 +1,166 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { UptideError } from '../../errors.js'; +import { minVersion } from '../../fetch/range.js'; +import { type GroundTruth, type PackVerification, truthDigest } from '../contract.js'; +import { packsDir } from './pack-test.js'; +import { packConstant, packDir, parseRegistry, renderRegistry } from './registry-file.js'; + +export interface ScaffoldOptions { + /** The uptide checkout. */ + root: string; + package: string; + /** Semver ranges: installed versions migrated from, target versions. */ + from: string; + to: string; + /** A GitHub handle, recorded as `meta.maintainer`. */ + maintainer?: string; +} + +export interface ScaffoldResult { + dir: string; + constant: string; + /** Repository-relative paths, written in this order. */ + files: string[]; + /** Whether the checkout's biome formatted them (it is what `pnpm lint` checks). */ + formatted: boolean; +} + +const RULE = 'example-rename'; + +function indexFile(o: ScaffoldOptions, constant: string): string { + const pkg = JSON.stringify(o.package).slice(1, -1).replaceAll("'", "\\'"); + return `import { definePack, replaceAtSite } from '../contract.js'; + +/** + * ${o.package} ${o.from} → ${o.to}. Every rule and note says where it comes from (\`meta.sources\`), + * has a fixture in \`fixtures/\`, and is scored against \`ground-truth.json\` by + * \`uptide pack test ${o.package}\`. See docs/packs.md. + */ +export const ${constant} = definePack({ + meta: { + package: '${pkg}', + from: '${o.from}', + to: '${o.to}', + // The changelog or migration guide every rule below is taken from. + sources: [{ title: '${pkg} on npm', url: 'https://www.npmjs.com/package/${o.package}' }], + maintainer: '${o.maintainer ?? 'uptide-dev'}', + }, + rules: [ + { + // An example to replace: an export renamed between the two majors. + id: '${RULE}', + summary: 'oldName was renamed newName', + severity: 'breaking', + // The findings \`check\` reports for it: the type diff sees \`oldName\` removed. + kinds: ['removed', 'renamed'], + symbols: /^oldName$/, + guide: 'oldName was renamed newName, with the same arguments and result.', + rewrite: (text, finding) => replaceAtSite(text, finding, 'oldName', 'newName'), + }, + ], + behavior: [], + instructions: 'Migrate only the reported site, from the compiler error and the guide above.', +}); +`; +} + +const BEFORE = `import { oldName } from 'PKG'; // @uptide ${RULE} at:oldName + +export const value = oldName(1); // @uptide ${RULE} at:oldName +// A different name that only starts the same: left alone. +export const oldNameLabel = 'kept'; // @uptide ${RULE} keep at:oldNameLabel +`; + +function testFile(o: ScaffoldOptions, constant: string): string { + return `import { fileURLToPath } from 'node:url'; +import { expect, it } from 'vitest'; +import { runFixtures } from '../tooling/fixtures.js'; +import { ${constant} } from './index.js'; + +it('${o.package.replaceAll("'", "\\'")}: every fixture rewrites and detects exactly its marked sites', () => { + const result = runFixtures(${constant}, fileURLToPath(new URL('.', import.meta.url))); + expect(result.cases.length).toBeGreaterThan(0); + expect(result.unknown).toEqual([]); + expect(result.rewriteFailures).toEqual([]); + expect(result.falsePositives).toEqual([]); + expect(result.falseNegatives).toEqual([]); +}); +`; +} + +/** Writes the files only when none of them exists; the registry is rendered again in full. */ +export function scaffoldPack(options: ScaffoldOptions): ScaffoldResult { + if (!/^(?:@[a-z0-9][\w.-]*\/)?[a-z0-9][\w.-]*$/.test(options.package)) + throw new UptideError('INVALID_GROUND_TRUTH', `not an npm package name: ${options.package}`); + for (const [flag, range] of [ + ['--from', options.from], + ['--to', options.to], + ] as const) + if (minVersion(range) === undefined) + throw new UptideError('UNSUPPORTED_VERSION_RANGE', `${flag} ${range}: not a semver range`); + const packs = packsDir(options.root); + const registryFile = join(packs, 'registry.ts'); + if (!existsSync(registryFile)) + throw new UptideError( + 'NOT_UPTIDE_CHECKOUT', + `no pack registry at ${registryFile}: run uptide pack new inside an uptide checkout`, + ); + const dir = packDir(options.package); + const constant = packConstant(options.package); + const entries = parseRegistry(readFileSync(registryFile, 'utf8')); + const home = join(packs, dir); + if (existsSync(home) || entries.some((e) => e.dir === dir || e.name === constant)) + throw new UptideError('PACK_EXISTS', `a pack for ${options.package} already exists: ${home}`); + + const truth: GroundTruth = { + $comment: `Public repositories at the commit before they upgraded ${options.package} from ${options.from} to ${options.to}, and every finding the pack must report there. See docs/packs.md, "Ground truth".`, + package: options.package, + repos: [], + }; + const before = BEFORE.replace('PKG', options.package); + const after = before + .replace('import { oldName }', 'import { newName }') + .replace('= oldName(1)', '= newName(1)'); + const files: [string, string][] = [ + [join(home, 'index.ts'), indexFile(options, constant)], + [join(home, 'fixtures', RULE, 'before.ts'), before], + [join(home, 'fixtures', RULE, 'after.ts'), after], + [join(home, `${dir}.test.ts`), testFile(options, constant)], + [join(home, 'ground-truth.json'), `${JSON.stringify(truth, null, 2)}\n`], + ]; + for (const [file, text] of files) { + mkdirSync(join(file, '..'), { recursive: true }); + writeFileSync(file, text); + } + writeFileSync(registryFile, renderRegistry([...entries, { dir, name: constant }])); + const written = [...files.map(([file]) => file), registryFile]; + // The checkout's own formatter has the last word on layout, so `pnpm lint` passes as is. + // Fixtures are test data it never touches: their text is the test. + const biome = join(options.root, 'node_modules', '.bin', 'biome'); + const formattable = written.filter((file) => !file.includes(`${dir}/fixtures/`)); + const formatted = + existsSync(biome) && + spawnSync(biome, ['format', '--write', ...formattable], { + cwd: options.root, + stdio: 'ignore', + }).status === 0; + // What `pack test` records for a pack with no ground-truth repository yet, tied to the + // ground-truth file as formatted: the first run finds it current. + const verification: PackVerification = { + status: 'candidate', + truth: truthDigest(readFileSync(join(home, 'ground-truth.json'), 'utf8')), + repos: 0, + breaking: { precision: 1, recall: 1, falsePositives: 0 }, + }; + const verificationFile = join(home, 'verification.json'); + writeFileSync(verificationFile, `${JSON.stringify(verification, null, 2)}\n`); + written.splice(written.length - 1, 0, verificationFile); + return { + dir, + constant, + files: written.map((file) => relative(options.root, file)), + formatted, + }; +} diff --git a/packages/core/src/packs/tooling/truth.ts b/packages/core/src/packs/tooling/truth.ts new file mode 100644 index 0000000..ada7d16 --- /dev/null +++ b/packages/core/src/packs/tooling/truth.ts @@ -0,0 +1,204 @@ +import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { defaultCacheDir } from '../../cache/paths.js'; +import { UptideError } from '../../errors.js'; +import { packageManager } from '../../fix/managers/manager.js'; +import { command, git } from '../../fix/process.js'; +import type { GroundTruth, GroundTruthRepo } from '../contract.js'; + +/** + * Ground-truth repositories are fetched once and kept: a shallow fetch of the one pinned + * commit, dependencies installed with lifecycle scripts off, nothing from the repository ever + * executed (no hook, no script; the same environment `fix` installs with). After the first + * run `uptide pack test` needs no network but the registry tarballs `check` caches itself. + */ +export function truthCacheDir(env: NodeJS.ProcessEnv = process.env): string { + return env.UPTIDE_GROUND_TRUTH_CACHE || join(defaultCacheDir(env), 'ground-truth'); +} + +/** What a corpus or ground-truth entry names: a public repository at a commit. */ +export interface PinnedRepo { + repo: string; + commit: string; +} + +const SHA = /^[0-9a-f]{40}$/; +const REPO = /^[\w.-]+\/[\w.-]+$/; + +export function label(entry: GroundTruthRepo): string { + return entry.fixture ?? `${entry.repo}@${(entry.commit ?? '').slice(0, 12)}`; +} + +/** Everything wrong with a ground-truth file, as sentences: `pack test` refuses to score it. */ +export function truthProblems(truth: GroundTruth, packageName: string): string[] { + const problems: string[] = []; + if (truth.package !== packageName) + problems.push(`package is "${truth.package}", the pack is "${packageName}"`); + truth.repos.forEach((r, i) => { + const where = `repos[${i}]`; + if (r.fixture === undefined) { + if (!r.repo || !REPO.test(r.repo)) problems.push(`${where}: repo must be owner/name`); + if (!r.commit || !SHA.test(r.commit)) + problems.push(`${where}: commit must be a full 40-character SHA`); + } else if (r.repo || r.commit) problems.push(`${where}: a fixture has no repo or commit`); + if (!r.from || !r.to) problems.push(`${where}: from and to are exact versions`); + for (const f of r.findings ?? []) + if (!f.file || !Number.isInteger(f.line) || f.line < 1 || !f.rule) + problems.push(`${where}: every finding has a file, a line and a rule`); + }); + return problems; +} + +export function cachedRepoDir(entry: PinnedRepo, cacheDir = truthCacheDir()): string { + return join(cacheDir, `${entry.repo.replace('/', '__')}@${entry.commit}`); +} + +/** What a failed install said, in a few lines: the ones that say why, not the usage text. */ +function why(output: string): string { + const lines = output.split('\n').map((l) => l.trim()); + const reasons = lines.filter((l) => /ERR|error .*\S|Missing|not in sync|can only|ENOENT/.test(l)); + return (reasons.length ? reasons.slice(0, 4) : lines.filter(Boolean).slice(-4)).join(' | '); +} + +/** Non-frozen installs, scripts still off: for a lockfile its own repository left out of sync. */ +const UNFROZEN: Record = { + npm: ['install', '--ignore-scripts', '--no-audit', '--no-fund'], + pnpm: ['install', '--ignore-scripts', '--no-frozen-lockfile'], + 'yarn-classic': ['install', '--ignore-scripts', '--non-interactive'], + 'yarn-berry': ['install'], +}; + +/** + * The repository's own lockfile, frozen, with lifecycle scripts off. A repository whose + * lockfile was already out of sync at that commit (it happens: `npm ci` refuses what `npm + * install` accepted) is installed without freezing, scripts still off; `pack test` then + * reports when the version it scores is not the ground truth's `from`. + */ +async function installDependencies( + dir: string, + name: string, + options: EnsureOptions, +): Promise<{ manager: string; frozen: boolean }> { + const pm = packageManager(dir); + const env = { ...pm.env, npm_config_manage_package_manager_versions: 'true' }; + const timeout = options.timeoutMs ?? 900_000; + options.log?.(`installing ${name} with ${pm.kind}, scripts off`); + const result = await command(dir, pm.bin, pm.args, timeout, env); + if (result.code === 0) return { manager: pm.kind, frozen: true }; + const retry = UNFROZEN[pm.kind] as string[]; + const args = pm.kind === 'yarn-berry' ? [...retry, ...pm.args.slice(2)] : retry; + options.log?.(`the frozen install failed (${why(result.output)}); installing without freezing`); + const again = await command(dir, pm.bin, args, timeout, env); + if (again.code !== 0) + throw new UptideError( + 'GROUND_TRUTH_INSTALL', + `installing ${name} failed${again.timeout ? ' (timed out)' : ''}: ${why(again.output)}`, + ); + return { manager: pm.kind, frozen: false }; +} + +interface Marker { + repo: string; + commit: string; + installed: boolean; + manager?: string; +} + +const MARKER = '.uptide-ground-truth.json'; + +function readMarker(dir: string): Marker | undefined { + try { + return JSON.parse(readFileSync(join(dir, MARKER), 'utf8')) as Marker; + } catch { + return undefined; + } +} + +export interface EnsureOptions { + cacheDir?: string; + /** Never touch the network: a repository not in the cache is an error. */ + offline?: boolean; + /** Install dependencies (default true): `check` needs them to type-check the repository. */ + install?: boolean; + log?: (line: string) => void; + /** Install timeout per repository, ms. */ + timeoutMs?: number; +} + +/** + * The directory of a pinned public repository, fetched and installed on first use. The + * marker file is written last: a directory without it is an interrupted fetch and is + * fetched again. + */ +export async function ensureRepo(entry: PinnedRepo, options: EnsureOptions = {}): Promise { + const cacheDir = options.cacheDir ?? truthCacheDir(); + const dir = cachedRepoDir(entry, cacheDir); + const install = options.install ?? true; + const marker = readMarker(dir); + if (marker && (marker.installed || !install)) return dir; + if (options.offline) + throw new UptideError( + 'GROUND_TRUTH_NOT_CACHED', + `${entry.repo}@${entry.commit.slice(0, 12)} is not in the ground-truth cache (${cacheDir}); run without --offline once, or pnpm packs:fetch`, + ); + if (!SHA.test(entry.commit) || !REPO.test(entry.repo)) + throw new UptideError('INVALID_GROUND_TRUTH', `not a pinned public repository: ${entry.repo}`); + if (!marker) { + options.log?.(`fetching ${entry.repo}@${entry.commit.slice(0, 12)} (shallow)`); + const partial = `${dir}.partial-${process.pid}`; + rmSync(partial, { recursive: true, force: true }); + mkdirSync(partial, { recursive: true }); + try { + git(partial, 'init', '-q'); + git(partial, 'remote', 'add', 'origin', `https://github.com/${entry.repo}.git`); + git(partial, 'fetch', '-q', '--depth', '1', '--no-tags', 'origin', entry.commit); + git(partial, 'checkout', '-q', '--detach', 'FETCH_HEAD'); + const head = git(partial, 'rev-parse', 'HEAD'); + if (head !== entry.commit) throw new Error(`fetched ${head}, expected ${entry.commit}`); + rmSync(dir, { recursive: true, force: true }); + renameSync(partial, dir); + } catch (err) { + rmSync(partial, { recursive: true, force: true }); + throw new UptideError( + 'GROUND_TRUTH_FETCH', + `cannot fetch ${entry.repo}@${entry.commit}: ${(err as Error).message.split('\n')[0]}`, + ); + } + } + let installed = false; + let manager: string | undefined; + let frozen: boolean | undefined; + if (install) { + const result = await installDependencies(dir, entry.repo, options); + manager = result.manager; + frozen = result.frozen; + installed = true; + } + writeFileSync( + join(dir, MARKER), + `${JSON.stringify({ repo: entry.repo, commit: entry.commit, installed, ...(manager ? { manager, frozen } : {}) })}\n`, + ); + return dir; +} + +/** + * A fixture repository of this tree, installed in place (its `node_modules` is ignored by + * git). Fixtures are synthetic: they are scored, never counted toward `verified`. + */ +export async function ensureFixture( + root: string, + fixture: string, + options: EnsureOptions = {}, +): Promise { + const dir = join(root, fixture); + if (!existsSync(join(dir, 'package.json'))) + throw new UptideError('INVALID_GROUND_TRUTH', `no fixture repository at ${fixture}`); + if (existsSync(join(dir, 'node_modules')) || options.install === false) return dir; + if (options.offline) + throw new UptideError( + 'GROUND_TRUTH_NOT_CACHED', + `${fixture} has no node_modules; run without --offline once`, + ); + await installDependencies(dir, fixture, options); + return dir; +} diff --git a/scripts/pack-test-report.ts b/scripts/pack-test-report.ts new file mode 100644 index 0000000..0156a79 --- /dev/null +++ b/scripts/pack-test-report.ts @@ -0,0 +1,18 @@ +/** + * What `uptide pack test --json` wrote, in the words `uptide pack test` prints: CI keeps the + * JSON as an artifact and shows this in the log and the job summary. + * + * tsx scripts/pack-test-report.ts pack-test.json + */ +import { readFileSync } from 'node:fs'; +import type { PackTestReport } from '@uptide/core'; +import { formatPackTest } from '../packages/cli/src/pack.ts'; + +const file = process.argv[2] ?? 'pack-test.json'; +const result = JSON.parse(readFileSync(file, 'utf8')) as { + passed: boolean; + packs: PackTestReport[]; +}; +const out = result.packs.map((report) => formatPackTest(report)).join('\n'); +console.log(`\`\`\`\n${out}\n\`\`\``); +console.log(result.passed ? 'Every pack passed.' : 'At least one pack failed.'); diff --git a/scripts/packs-fetch.ts b/scripts/packs-fetch.ts new file mode 100644 index 0000000..95401ca --- /dev/null +++ b/scripts/packs-fetch.ts @@ -0,0 +1,49 @@ +/** + * Fetch and cache every public repository the packs' ground truth names, and the corpus + * (fixtures/corpus.json): a shallow fetch of the pinned commit, dependencies installed with + * lifecycle scripts off. After this, `uptide pack test --offline` needs no network for them. + * + * pnpm packs:fetch [pack...] [--corpus] [--no-install] + * + * The cache is ~/.cache/uptide/ground-truth (UPTIDE_GROUND_TRUTH_CACHE overrides it); CI keeps + * it between runs keyed on the ground-truth files. + */ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { ensureRepo, type PinnedRepo, registeredPacks, truthCacheDir } from '@uptide/core'; + +const args = process.argv.slice(2); +const names = args.filter((a) => !a.startsWith('--')); +const install = !args.includes('--no-install'); +const root = join(import.meta.dirname, '..'); + +const wanted: { repo: PinnedRepo; for: string }[] = []; +for (const entry of registeredPacks()) { + if (names.length && !names.includes(entry.pack.name) && !names.includes(entry.dir)) continue; + const truth = JSON.parse( + readFileSync(join(root, 'packages/core/src/packs', entry.dir, 'ground-truth.json'), 'utf8'), + ) as { repos: { repo?: string; commit?: string }[] }; + for (const r of truth.repos) + if (r.repo && r.commit) + wanted.push({ repo: { repo: r.repo, commit: r.commit }, for: entry.pack.name }); +} +if (args.includes('--corpus')) + for (const r of JSON.parse( + readFileSync(join(root, 'fixtures/corpus.json'), 'utf8'), + ) as PinnedRepo[]) + wanted.push({ repo: { repo: r.repo, commit: r.commit }, for: 'corpus' }); + +console.log(`cache: ${truthCacheDir()}`); +let failed = 0; +for (const w of wanted) { + try { + const dir = await ensureRepo(w.repo, { install, log: (line) => console.log(` ${line}`) }); + console.log(`✓ ${w.repo.repo}@${w.repo.commit.slice(0, 12)} (${w.for}) ${dir}`); + } catch (err) { + failed++; + console.log( + `✗ ${w.repo.repo}@${w.repo.commit.slice(0, 12)} (${w.for}) ${(err as Error).message}`, + ); + } +} +process.exit(failed ? 1 : 0); From 5b165146ecf805bb22957d213428927138f95d18 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 06:47:13 +0000 Subject: [PATCH 3/9] docs(packs): the contract, Write a pack, templates, the queue and the Packs workflow docs/packs.md with zod as the reference; CONTRIBUTING.md "Write a pack"; a pack pull request template and the pack request issue form; ADR 0015; packs/queue.json from public npm data (pnpm packs:queue); a Packs workflow that runs pack test for every pack with the ground-truth cache. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01B2T8GmEVnZwfNrugajEMqi Signed-off-by: Claude --- .changeset/packs-public-contract.md | 11 + .github/ISSUE_TEMPLATE/pack-request.yml | 41 +++- .github/PULL_REQUEST_TEMPLATE.md | 2 + .github/PULL_REQUEST_TEMPLATE/pack.md | 34 +++ .github/workflows/packs.yml | 67 ++++++ CONTRIBUTING.md | 77 +++++-- docs/cli.md | 27 ++- docs/decisions/0015-public-pack-contract.md | 28 +++ docs/packs.md | 215 ++++++++++++++++++ packs/queue.json | 231 ++++++++++++++++++++ scripts/packs-queue.ts | 150 +++++++++++++ 11 files changed, 859 insertions(+), 24 deletions(-) create mode 100644 .changeset/packs-public-contract.md create mode 100644 .github/PULL_REQUEST_TEMPLATE/pack.md create mode 100644 .github/workflows/packs.yml create mode 100644 docs/decisions/0015-public-pack-contract.md create mode 100644 docs/packs.md create mode 100644 packs/queue.json create mode 100644 scripts/packs-queue.ts diff --git a/.changeset/packs-public-contract.md b/.changeset/packs-public-contract.md new file mode 100644 index 0000000..6d38396 --- /dev/null +++ b/.changeset/packs-public-contract.md @@ -0,0 +1,11 @@ +--- +"uptide": minor +--- + +Migration packs are a public, testable contract (docs/packs.md). `uptide pack new` scaffolds +a pack (an example rule, a fixture pair, a test, empty ground truth) and registers it; +`uptide pack test` scores packs against their fixtures and against public repositories at the +commit before their upgrade, with precision and recall per rule, every false positive and +false negative, and `--json` for CI. A pack is labeled verified only with ground truth from at +least two public repositories and no false positive among its breaking findings; otherwise it +ships as a candidate and `check`, `list` and `fix` treat the dependency as generic. diff --git a/.github/ISSUE_TEMPLATE/pack-request.yml b/.github/ISSUE_TEMPLATE/pack-request.yml index 2c7e58a..2d720ff 100644 --- a/.github/ISSUE_TEMPLATE/pack-request.yml +++ b/.github/ISSUE_TEMPLATE/pack-request.yml @@ -1,23 +1,48 @@ -name: Dependency request -description: Ask for a migration pack for another dependency +name: Pack request +description: Ask for a migration pack for one dependency upgrade, or say you are writing one +title: 'pack: → ' labels: [pack] body: + - type: markdown + attributes: + value: | + A pack is accepted with its sources, fixtures and ground truth from public repositories (docs/packs.md). + The ranked list of what is worth a pack next is packs/queue.json. - type: input id: dependency attributes: - label: Dependency and upgrade - placeholder: openai 4 → 5 + label: Package and upgrade + description: The npm package and the two majors, as semver ranges + placeholder: ai >=6 <7 → >=7 <8 + validations: + required: true + - type: input + id: guide + attributes: + label: Migration guide or changelog + description: The official source every rule will be taken from + placeholder: https://ai-sdk.dev/docs/migration-guides/migration-guide-7-0 validations: required: true - type: textarea id: breaks attributes: label: What the upgrade breaks - description: The changes that cost you time, with the official migration guide if there is one + description: The changes that cost you time, and any that compile but behave differently validations: required: true - type: textarea - id: example + id: repos + attributes: + label: Public repositories that made this upgrade + description: Links to the commits or pull requests where they upgraded. Two are needed for a verified pack; the commit before each one is its ground truth. + placeholder: https://github.com/owner/name/pull/123 + - type: dropdown + id: writing attributes: - label: A public repository that still needs it - description: A pack is accepted with ground truth from a public repository + label: Are you writing it? + options: + - No, asking for it + - Yes, I am writing it (the queue entry becomes in-progress) + validations: + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d24fc50..21b92e9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,5 @@ + + ## What and why diff --git a/.github/PULL_REQUEST_TEMPLATE/pack.md b/.github/PULL_REQUEST_TEMPLATE/pack.md new file mode 100644 index 0000000..4f96341 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/pack.md @@ -0,0 +1,34 @@ + + +## Pack + + + +## Sources + + + +## Rules and behavior notes + + + +## Ground-truth repositories + + + +## `uptide pack test` output + +``` +paste the output of: pnpm uptide pack test +``` + +## Checklist + +- [ ] Every rule and note comes from a source listed in `meta.sources` +- [ ] Every rule that rewrites or detects has a fixture case, with a site it must leave alone (`keep`) +- [ ] Ground truth: public repositories at the commit before their upgrade; expected findings read from their own upgrade commit and the compiler, never copied from Uptide's output +- [ ] At least two public repositories for `verified`; otherwise the pack ships as a candidate +- [ ] `pnpm uptide pack test ` passes and its output is pasted above; `verification.json` written with `--write` +- [ ] `pnpm lint && pnpm typecheck && pnpm test` pass +- [ ] Changeset added (`pnpm changeset`): a new pack is something users notice +- [ ] No private code, names or paths anywhere: fixtures are synthetic, ground truth is public diff --git a/.github/workflows/packs.yml b/.github/workflows/packs.yml new file mode 100644 index 0000000..2d997c9 --- /dev/null +++ b/.github/workflows/packs.yml @@ -0,0 +1,67 @@ +name: Packs + +# `uptide pack test` for every pack: fixtures, and `check` on each ground-truth repository +# against the expected findings (docs/packs.md). It fails on a false positive among breaking +# findings, a failing fixture, or a verification.json the evidence no longer supports. +# +# The job always reports, so it can be a required check: on a pull request that touches +# neither a pack nor the engine that produces their findings it passes without running. + +on: + push: + branches: [main] + pull_request: + +jobs: + pack-test: + name: Pack test + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Touches packs + id: scope + env: + BASE: ${{ github.event.pull_request.base.sha }} + run: | + if [ "${{ github.event_name }}" != "pull_request" ]; then echo "run=true" >> "$GITHUB_OUTPUT"; exit 0; fi + changed=$(git diff --name-only "$BASE" HEAD) + echo "$changed" + if echo "$changed" | grep -Eq '^(packages/core/src/(packs|check|adapters|fix/plan\.ts|fix/report\.ts)|packages/core/package\.json|packs/|fixtures/repos/storefront/|scripts/pack-test-report\.ts|\.github/workflows/packs\.yml|pnpm-lock\.yaml)'; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "No pack, and nothing their findings come from, changed: nothing to score." + echo "run=false" >> "$GITHUB_OUTPUT" + fi + - uses: pnpm/action-setup@v4 + if: steps.scope.outputs.run == 'true' + - uses: actions/setup-node@v4 + if: steps.scope.outputs.run == 'true' + with: + node-version: 22 + cache: pnpm + - if: steps.scope.outputs.run == 'true' + run: pnpm install --frozen-lockfile --ignore-scripts + # Ground-truth repositories (shallow, installed with scripts off) and the registry + # tarballs `check` extracts. Keyed on the ground truth: a new repository is fetched once. + - uses: actions/cache@v4 + if: steps.scope.outputs.run == 'true' + with: + path: ~/.cache/uptide + key: packs-${{ runner.os }}-${{ hashFiles('packages/core/src/packs/*/ground-truth.json') }} + restore-keys: packs-${{ runner.os }}- + - name: uptide pack test + if: steps.scope.outputs.run == 'true' + run: | + set +e + pnpm -s uptide pack test --json > pack-test.json + code=$? + node --conditions=source --import tsx scripts/pack-test-report.ts pack-test.json | tee -a "$GITHUB_STEP_SUMMARY" + exit $code + - uses: actions/upload-artifact@v4 + if: always() && steps.scope.outputs.run == 'true' + with: + name: pack-test + path: pack-test.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index edc9837..8f9977c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,6 +35,7 @@ Other suites, when your change touches them: | `UPTIDE_NETWORK=1 pnpm test` | real package digests, the Stripe `tsc` fixture | npm registry | | `pnpm smoke 20` / `pnpm smoke 22` | the packed CLI on npm, pnpm and Yarn fixtures | Docker | | `pnpm eval:check fixtures/repos/storefront --truth=fixtures/truth/storefront.json` | check against the errors a real upgrade produced | the fixture's dependencies (`pnpm install --ignore-scripts` in it) | +| `pnpm uptide pack test` | every pack against its fixtures and its public ground truth | network the first time (`pnpm packs:fetch`), then `--offline` | ## Layout @@ -69,25 +70,73 @@ from this repository, so it is scanned in full. Which scan runs when: ## How packs work A migration pack (`packages/core/src/packs//`) owns the knowledge of one -dependency; the runner owns everything else (install, verification, commits, publishing). -The contract is `MigrationPack` in `packages/core/src/packs/types.ts`. A pack provides: - -- **rules**: deterministic rewrites, applied only at sites `check` reported; -- **a guide** for assisted fixes, and a validator that can reject a patch the compiler - would accept (a fallback like `?? 0` that hides a missing value, for example); -- **findings the compiler cannot see** (a Stripe client without `apiVersion`); -- **review material**: decisions that are the user's to make, changelog entries filtered to - what the code uses, test follow-ups. +dependency upgrade; the runner owns everything else (install, verification, commits, +publishing). The public contract, with zod as the reference, is [docs/packs.md](docs/packs.md) +and `packages/core/src/packs/contract.ts`. A pack provides: + +- **metadata**: the package, the `from` and `to` version ranges, its sources (changelog, + migration guide) and its maintainer; +- **rules**: what `check` reports that the rule claims, and a deterministic rewrite applied + only at reported sites, or a guide for the agent; +- **behavior notes**: changes the compiler cannot see (a Stripe client without + `apiVersion`, zod's reworded default messages), and how each is reported; +- **agent instructions** for what no rule can do; +- **fixtures** and **ground truth**: public repositories at the commit before their upgrade, + with the findings expected there. Rules for a pack change: 1. A rule never edits a site that was not reported, and never changes behavior silently. When behavior can differ, the report says so and asks. -2. Every rule has a test with the code before and after, and a case it must leave alone. -3. A claim about what a pack finds is backed by ground truth in `fixtures/truth/`: the - compiler errors a real upgrade produced on a fixture in this repository. No private - code, paths or names anywhere in the tree: fixtures are synthetic. -4. Server API changes (Stripe API versions) are never rewritten by rule. +2. Every rule that rewrites or detects has a fixture with the code before and after, and a + site it must leave alone. +3. A claim about what a pack finds is backed by ground truth from public repositories, scored + by `uptide pack test`. Expected findings come from the repository's own upgrade and the + compiler, never from Uptide's output. No private code, paths or names anywhere: fixtures + are synthetic, ground truth is public. +4. A pack is verified only with ground truth from two public repositories and no false + positive among its breaking findings; until then it ships as a candidate, and the CLI + treats the dependency as generic. +5. Server API changes (Stripe API versions) are never rewritten by rule. + +## Write a pack + +About an hour for a focused upgrade. Pick one from [`packs/queue.json`](packs/queue.json), or +open a **Pack request** issue, and say you are writing it. + +1. **Read the sources.** The official migration guide and the changelog of the target major. + List the breaking changes, and for each: does the compiler see it, and is a rewrite safe + whatever the surrounding code is? (15 minutes) +2. **Scaffold.** + ```sh + pnpm uptide pack new --from ">=6 <7" --to ">=7 <8" --maintainer @you + ``` + This writes `packages/core/src/packs//` (the pack with one example rule, a fixture + pair, a test, an empty `ground-truth.json`, `verification.json`) and registers it. It + passes `pnpm lint` and `pnpm test` as written. +3. **Write the rules** in `index.ts`, replacing the example: put the guide and changelog URLs + in `meta.sources`; one rule per compiler-visible change you can rewrite safely + (`kinds`, `symbols`, `rewrite`; `replaceAtSite` covers renames); a rule with only a `guide` + for one the agent must do; a behavior note for each change the compiler cannot see, with + `detect` when its sites can be found in the source. (15 minutes) +4. **Write fixtures**: `fixtures//before.ts` with each site marked + (`// @uptide `, `keep` for one it must leave alone) and `after.ts`, what the rules + must produce. `pnpm uptide pack test --fixtures-only` runs them in seconds. +5. **Find ground truth.** Two public repositories that made this upgrade in a commit you + can point at. Pin the commit before it; the expected findings are the sites the upgrade + commit changed for this package (`git diff -U0`, old line numbers), + each with its rule (`generic` when no rule of the pack covers it). (15 minutes) +6. **Score it.** + ```sh + pnpm uptide pack test # fetches the repositories once, scripts off + pnpm uptide pack test --write # records the status in verification.json + ``` + Every false positive among breaking findings fails it. Fix the rule, not the ground truth. + A false negative is a site to cover or a known gap to say in the pull request. +7. **Open the pull request** with the pack template (`?template=pack.md`), with the + `pack test` output pasted in, and a changeset. + +`pnpm uptide` runs the CLI from source, so nothing needs building while you work. ## Rules that do not bend diff --git a/docs/cli.md b/docs/cli.md index c79b6a0..65e4ab3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -88,8 +88,10 @@ or incomplete analysis. A failing package/workspace does not erase successful re ### Tiers -Every row says `verified` or `generic`. Verified: a migration pack covers the upgrade. -Generic: no pack; a finding is breaking only with evidence, which `--details` names under +Every row says `verified` or `generic`. Verified: a migration pack covers the upgrade, and +its ground truth from at least two public repositories has no false positive among breaking +findings ([docs/packs.md](packs.md)). A pack that has not met that bar yet is a candidate and +the row says generic. Generic: no verified pack; a finding is breaking only with evidence, which `--details` names under each site: - your code does not compile against the target at that site; @@ -312,6 +314,27 @@ and type verification on the recorded commit without installing or changing sour Diffs, file lists, verification and run details are collapsed; the Action comment and terminal share the compact migration renderer. +## `uptide pack new|test` (contributors) + +Packs are written in an uptide checkout; the contract is [docs/packs.md](packs.md) and the +steps are in [CONTRIBUTING.md](../CONTRIBUTING.md#write-a-pack). `pnpm uptide` runs the CLI +from source. + +```sh +pnpm uptide pack new ai --from ">=6 <7" --to ">=7 <8" --maintainer @you +pnpm uptide pack test ai # fixtures, then check on each ground-truth repository +pnpm uptide pack test --json # every pack, for CI +pnpm uptide pack test ai --offline # cached repositories only +pnpm uptide pack test ai --fixtures-only +pnpm uptide pack test ai --write # record the measured status in verification.json +``` + +`pack test` prints precision and recall per rule, overall and for breaking findings, and +every false positive and false negative as `repository file:line rule`. Exit codes: **0** +every pack passed, **1** a false positive among breaking findings, a failing fixture, or a +`verification.json` the run does not support, **2** it could not run (not in a checkout, +bad arguments). + ## `uptide telemetry on|off|status|show` Anonymous telemetry is off by default. The first interactive run asks once, default diff --git a/docs/decisions/0015-public-pack-contract.md b/docs/decisions/0015-public-pack-contract.md new file mode 100644 index 0000000..df51813 --- /dev/null +++ b/docs/decisions/0015-public-pack-contract.md @@ -0,0 +1,28 @@ +# Packs are a public contract, verified against public repositories + +Status: accepted, 2026-10-07. Builds on 0011 (verified packs) and 0012 (generic tier). + +Two packs were written by the people who wrote the runner, and their claims rested on one +synthetic fixture. More packs need a contract someone else can meet and a test someone else +can run, and the `verified` label has to mean the same thing for every pack. + +- The contract is code (`packages/core/src/packs/contract.ts`): metadata with sources, rules + that claim `check`'s findings and rewrite one reported site, behavior notes for what the + compiler cannot see with how each is reported, agent instructions, fixtures, ground truth. + `definePack` derives the runner's hooks from that alone; zod and stripe implement the same + interface by hand and keep their outputs. +- Ground truth is public repositories at the commit before their upgrade, with expected + findings read from their own upgrade and the compiler. Scoring runs `check` with the pack, + the same code path users get, and compares the plan's sites per rule. +- `verified` needs two public repositories and no false positive among breaking findings. + Recall is reported, not gated: a pack that finds less is still safe to run, a pack that + claims something false is not. Below the bar a pack ships as a candidate and the CLI treats + the dependency as generic. +- The CLI reads the status from a committed record (`verification.json`) tied to the digest + of the ground truth; CI runs `pack test` and fails when the record no longer matches. The + CLI never fetches ground truth at runtime. + +Consequences accepted: scoring needs the ground-truth repositories installed (with scripts +off), which costs network and disk once and is cached; a pack's rule ids for sites it does +not own are `generic`; a change to the engine can change a pack's score, which is why the +Packs workflow also runs when `check` changes. diff --git a/docs/packs.md b/docs/packs.md new file mode 100644 index 0000000..ddabab0 --- /dev/null +++ b/docs/packs.md @@ -0,0 +1,215 @@ +# Migration packs: the public contract + +A migration pack is everything Uptide knows about upgrading one dependency across one +breaking change. This page is the contract a pack meets, with the zod pack +(`packages/core/src/packs/zod/`) as the reference. Anyone can write a pack, and so can a +scheduled agent: `uptide pack new` scaffolds one and `uptide pack test` scores it. The +contract is checked in code (`packages/core/src/packs/contract.ts`), not only described here. + +The step-by-step guide is in [CONTRIBUTING.md, "Write a pack"](../CONTRIBUTING.md#write-a-pack). +The next packages worth a pack are in [`packs/queue.json`](../packs/queue.json). + +## What a pack is + +A pack is a directory `packages/core/src/packs//`: + +| File | What it holds | +| --- | --- | +| `index.ts` | the pack: metadata, rules, behavior notes, agent instructions | +| `fixtures//before.ts`, `after.ts` | code before and after the rules, with the sites marked | +| `.test.ts` | runs the fixtures under `pnpm test` (scaffolded) | +| `ground-truth.json` | real public repositories at a pinned commit, with the findings expected there | +| `verification.json` | what `uptide pack test --write` measured; the status the CLI shows | + +It is registered in `packages/core/src/packs/registry.ts`, which `uptide pack new` writes. + +### Metadata + +```ts +meta: { + package: 'zod', + from: '>=3 <4', // installed versions it migrates from (semver range) + to: '>=4 <5', // target versions + sources: [{ title: 'Zod 4 migration guide', url: 'https://zod.dev/v4/changelog' }], + maintainer: 'uptide-dev', +}, +``` + +`sources` are the changelog and migration guide every rule and note is taken from. A rule +that no source supports does not belong in a pack. `supports(from, to)` defaults to the two +ranges; a pack may narrow it (stripe also requires the target to be newer than what is +installed). + +### Mechanical rules: detect and rewrite + +```ts +{ + id: 'error-params', + summary: 'required_error and invalid_type_error become one error callback', + severity: 'breaking', // or 'deprecated' + kinds: ['signature', 'type', 'required'], + symbols: /string|number|.../, // detect: the findings `check` reports that it claims + guide: zodGuide.errors, // what the agent is told at a site the rule cannot rewrite + rewrite: (text, finding) => ..., // the edit at one reported site, or { applied: false, reason } +} +``` + +- **Detect.** `check` already diffs the two versions' declarations and compiles the + repository against the target. A rule claims the findings whose change kind and symbol + path it matches (`kinds`, `symbols`). For sites no type diff reports, a rule can add + `detect(text, file)`, which returns sites found in the source of a file that uses the + package; `check` lists them as the pack's own findings. +- **Rewrite.** `rewrite(text, finding, context)` edits the reported site and nothing else, + or declines with a reason (`existing error/errorMap needs manual merging`). A rule without + `rewrite` is assisted: the agent gets its `guide`, and a patch is kept only when the + site's compiler error disappears and no new one appears. +- A rule never changes behavior silently. When behavior can differ, it is a behavior note + (below), reported and left to a person. + +`definePack` (in `contract.ts`) builds the runner's hooks from rules and notes alone, which +is what `uptide pack new` scaffolds. `replaceAtSite(text, finding, before, after)` is the +common rewrite: one identifier at the reported site. Zod and Stripe implement the same +interface by hand because they gather context no rule has (default messages, API versions). + +### Fixtures + +Every rule that rewrites or detects has fixtures, and every fixture case has a site it must +leave alone. A site is a line that ends in a marker comment: + +```ts +export const name = z.string({ required_error: 'Name is required' }); // @uptide error-params at:z.string +export const merged = z.string({ required_error: 'x', errorMap: m }); // @uptide error-params keep at:z.string +export const contact = z.string().email(); // @uptide string-format at:.email( path:ZodString#email +``` + +- `keep`: the rule must leave this site alone (a rewrite must decline, `detect` must not find it). +- `at:`: where on the line the site starts, as `check` would report it (default: the + first non-blank column). +- `kind:`, `path:`: the change `check` reports there, when the rule matches on + them (default: the rule's first kind, and the `at:` text). + +Each marked site of a rule with `rewrite` is rewritten, bottom-up, and the result must equal +`after.ts` byte for byte (the markers are comments, so they stay). A rule or note with +`detect` must find exactly the marked lines in `before.ts`, and nothing on any other line. +Fixtures are test data: biome and `tsc` skip them, and they are never reformatted. + +### Behavior notes: what the compiler cannot see + +```ts +behavior: [ + { + id: 'default-messages', + summary: 'Zod 4 words its default error messages differently; code and tests matching the old text keep compiling', + reported: ['decision', 'test-follow-up'], + }, +], +``` + +A behavior note is a change where the same code compiles and does something else. How a +site is reported is part of the contract (`reported`): + +- `finding`: `check` lists the site as the pack's own finding (evidence `pack`), found by + the note's `detect`; `fix` never edits it unasked. Example: a Stripe client created + without `apiVersion`, whose API version the SDK bump changes at runtime. +- `decision`: the pull request lists it under "Decisions for you", with the files and lines. +- `test-follow-up`: an assertion is updated only after the migrated code made that test + fail, and the test then has to pass (zod's reworded default messages). + +A note found by `detect` is scored by `pack test` like a rule. + +### Agent instructions + +`instructions` is what the agent is told at a site no rule rewrites, after the rule's own +`guide`. They come from the sources, say what must not change (values, messages, control +flow), and never ask for a cast or a suppressed diagnostic to make the compiler pass. +Runtime never downloads instructions: they are checked in, with the date they were taken. + +## Ground truth + +```json +{ + "package": "zod", + "repos": [ + { + "repo": "owner/name", + "commit": "", + "migration": "https://github.com/owner/name/commit/", + "from": "3.25.76", + "to": "4.1.12", + "why": "what this repository exercises", + "findings": [{ "file": "src/schema.ts", "line": 12, "rule": "error-params" }] + } + ] +} +``` + +Each entry is a public repository at the commit **before** it made the upgrade, and the +findings the pack must report there: repository-relative file, 1-based line, and the rule +or note id. `from` is the version the lockfile has at that commit, and `to` the version the +repository upgraded to. + +Where the expected findings come from: + +1. the sites the repository's own upgrade commit changed for this dependency (`git diff + -U0`, the old side's line numbers), and +2. the errors the compiler reports at the target that the upgrade commit fixed, + +each read and kept only when it is about this dependency. A site that `check` reports and +that nobody had to change is a false positive, and it stays one: the expected findings +never come from Uptide's own output. A site the pack has no rule for is expected under +`generic`, which is what `pack test` calls a site the plan groups under no rule of the pack. + +`fixture` entries (`"fixture": "fixtures/repos/storefront"`) are repositories in this tree. +They are scored like the others and never count toward `verified`. + +`uptide pack test` fetches each repository once: a shallow fetch of the pinned commit only, +dependencies installed with lifecycle scripts off, and nothing from the repository is ever +executed. They are kept in `~/.cache/uptide/ground-truth` (`UPTIDE_GROUND_TRUTH_CACHE`), so +`pack test --offline` runs without network after the first time. `pnpm packs:fetch` fills +the cache for every pack (and `--corpus` for the repositories in `fixtures/corpus.json`). + +## Scoring: `uptide pack test` + +```sh +pnpm uptide pack test zod # from source: no build needed +uptide pack test --json # every pack, for CI +uptide pack test zod --fixtures-only # no repository +uptide pack test zod --write # record the result in verification.json +``` + +For each ground-truth repository it runs `check` with the pack (the same code path users +get), takes the sites of the plan for the package, and compares them with the expected +findings: + +- **per rule**: a site counts for a rule when file, line and rule all match; +- **overall**: at the line level, every severity; +- **breaking**: at the line level, breaking findings only. + +It prints precision and recall for each, and every false positive and false negative as +`repo file:line rule`, and sites found under another rule. It exits non-zero on any false +positive among breaking findings, any fixture failure, or a `verification.json` that no +longer matches what it measured (`--write` records it). + +## Verified or candidate + +A pack is **verified** only with ground truth from at least two public repositories and no +false positive among its breaking findings there (precision 100%), with at least one +breaking finding predicted. Anything else is a **candidate**: + +- it ships in the tree, and `pack test` scores it; +- `check`, `list` and `fix` treat the dependency as generic: no `verified` label, the + generic tier's evidence rule, the agent alone in `fix`. + +The CLI reads the status from `verification.json`, which `pack test --write` writes and +which records the digest of the ground-truth file it was measured against. CI runs +`pack test` for every pack on every pull request that touches packs and fails when the +record no longer matches the evidence, so the label cannot outlive what supports it. + +## The registry, the queue, CI + +- `packages/core/src/packs/registry.ts` is generated: `pack new` renders it again with the + new entry. Every pack in it is scored by CI; only verified ones are used by the CLI. +- `packs/queue.json` ranks the next packages worth a pack from public npm data + (`pnpm packs:queue`): weekly downloads times breaking majors in the last two years. +- The **Packs** workflow (`.github/workflows/packs.yml`) runs `uptide pack test --json` for + every pack with the ground-truth cache restored, on pull requests that touch packs. diff --git a/packs/queue.json b/packs/queue.json new file mode 100644 index 0000000..b8c8a74 --- /dev/null +++ b/packs/queue.json @@ -0,0 +1,231 @@ +{ + "$comment": "Generated by `pnpm packs:queue` from public npm data: weekly downloads (registry search) times breaking majors in the last two years (registry release dates). Pick one, say so in a pack-request issue, and follow CONTRIBUTING.md, \"Write a pack\".", + "generated": "2026-10-07", + "queue": [ + { + "package": "undici", + "from": ">=7 <8", + "to": ">=8 <9", + "why": "2 breaking majors in two years (7, 8), 247,722,467 downloads a week", + "status": "todo", + "weeklyDownloads": 247722467, + "breakingMajors2y": 2 + }, + { + "package": "zod", + "from": ">=3 <4", + "to": ">=4 <5", + "why": "1 breaking major in two years (4), 387,371,526 downloads a week", + "status": "verified", + "weeklyDownloads": 387371526, + "breakingMajors2y": 1 + }, + { + "package": "google-auth-library", + "from": ">=10 <11", + "to": ">=11 <12", + "why": "2 breaking majors in two years (10, 11), 119,280,995 downloads a week", + "status": "todo", + "weeklyDownloads": 119280995, + "breakingMajors2y": 2 + }, + { + "package": "@vitejs/plugin-react", + "from": ">=5 <6", + "to": ">=6 <7", + "why": "2 breaking majors in two years (5, 6), 116,919,148 downloads a week", + "status": "todo", + "weeklyDownloads": 116919148, + "breakingMajors2y": 2 + }, + { + "package": "react", + "from": ">=18 <19", + "to": ">=19 <20", + "why": "1 breaking major in two years (19), 224,422,510 downloads a week", + "status": "todo", + "weeklyDownloads": 224422510, + "breakingMajors2y": 1 + }, + { + "package": "react-dom", + "from": ">=18 <19", + "to": ">=19 <20", + "why": "1 breaking major in two years (19), 211,621,049 downloads a week", + "status": "todo", + "weeklyDownloads": 211621049, + "breakingMajors2y": 1 + }, + { + "package": "framer-motion", + "from": ">=13 <14", + "to": ">=14 <15", + "why": "3 breaking majors in two years (12, 13, 14), 58,492,608 downloads a week", + "status": "todo", + "weeklyDownloads": 58492608, + "breakingMajors2y": 3 + }, + { + "package": "next", + "from": ">=15 <16", + "to": ">=16 <17", + "why": "2 breaking majors in two years (15, 16), 76,902,421 downloads a week", + "status": "todo", + "weeklyDownloads": 76902421, + "breakingMajors2y": 2 + }, + { + "package": "react-router", + "from": ">=7 <8", + "to": ">=8 <9", + "why": "2 breaking majors in two years (7, 8), 68,732,660 downloads a week", + "status": "todo", + "weeklyDownloads": 68732660, + "breakingMajors2y": 2 + }, + { + "package": "validate-npm-package-name", + "from": ">=7 <8", + "to": ">=8 <9", + "why": "2 breaking majors in two years (7, 8), 58,734,038 downloads a week", + "status": "todo", + "weeklyDownloads": 58734038, + "breakingMajors2y": 2 + }, + { + "package": "@hookform/resolvers", + "from": ">=4 <5", + "to": ">=5 <6", + "why": "2 breaking majors in two years (4, 5), 58,680,661 downloads a week", + "status": "todo", + "weeklyDownloads": 58680661, + "breakingMajors2y": 2 + }, + { + "package": "protobufjs", + "from": ">=7 <8", + "to": ">=8 <9", + "why": "1 breaking major in two years (8), 110,035,263 downloads a week", + "status": "todo", + "weeklyDownloads": 110035263, + "breakingMajors2y": 1 + }, + { + "package": "ts-morph", + "from": ">=27 <28", + "to": ">=28 <29", + "why": "4 breaking majors in two years (25, 26, 27, 28), 35,821,740 downloads a week", + "status": "todo", + "weeklyDownloads": 35821740, + "breakingMajors2y": 3 + }, + { + "package": "ai", + "from": ">=6 <7", + "to": ">=7 <8", + "why": "4 breaking majors in two years (4, 5, 6, 7), 34,491,532 downloads a week", + "status": "todo", + "weeklyDownloads": 34491532, + "breakingMajors2y": 3 + }, + { + "package": "got", + "from": ">=15 <16", + "to": ">=16 <17", + "why": "2 breaking majors in two years (15, 16), 49,967,370 downloads a week", + "status": "todo", + "weeklyDownloads": 49967370, + "breakingMajors2y": 2 + }, + { + "package": "@angular-devkit/core", + "from": ">=21 <22", + "to": ">=22 <23", + "why": "4 breaking majors in two years (19, 20, 21, 22), 32,083,353 downloads a week", + "status": "todo", + "weeklyDownloads": 32083353, + "breakingMajors2y": 3 + }, + { + "package": "motion", + "from": ">=13 <14", + "to": ">=14 <15", + "why": "4 breaking majors in two years (11, 12, 13, 14), 28,260,335 downloads a week", + "status": "todo", + "weeklyDownloads": 28260335, + "breakingMajors2y": 3 + }, + { + "package": "@angular-devkit/schematics", + "from": ">=21 <22", + "to": ">=22 <23", + "why": "4 breaking majors in two years (19, 20, 21, 22), 27,747,217 downloads a week", + "status": "todo", + "weeklyDownloads": 27747217, + "breakingMajors2y": 3 + }, + { + "package": "stripe", + "from": ">=22 <23", + "to": ">=23 <24", + "why": "6 breaking majors in two years (18, 19, 20, 21, 22, 23), 27,492,363 downloads a week", + "status": "verified", + "weeklyDownloads": 27492363, + "breakingMajors2y": 3 + }, + { + "package": "zustand", + "from": ">=4 <5", + "to": ">=5 <6", + "why": "1 breaking major in two years (5), 71,534,632 downloads a week", + "status": "todo", + "weeklyDownloads": 71534632, + "breakingMajors2y": 1 + }, + { + "package": "recharts", + "from": ">=2 <3", + "to": ">=3 <4", + "why": "1 breaking major in two years (3), 71,292,254 downloads a week", + "status": "todo", + "weeklyDownloads": 71292254, + "breakingMajors2y": 1 + }, + { + "package": "proxy-agent", + "from": ">=7 <8", + "to": ">=8 <9", + "why": "2 breaking majors in two years (7, 8), 34,031,555 downloads a week", + "status": "todo", + "weeklyDownloads": 34031555, + "breakingMajors2y": 2 + }, + { + "package": "sonner", + "from": ">=1 <2", + "to": ">=2 <3", + "why": "1 breaking major in two years (2), 63,500,483 downloads a week", + "status": "todo", + "weeklyDownloads": 63500483, + "breakingMajors2y": 1 + }, + { + "package": "jest-cli", + "from": ">=29 <30", + "to": ">=30 <31", + "why": "1 breaking major in two years (30), 56,018,663 downloads a week", + "status": "todo", + "weeklyDownloads": 56018663, + "breakingMajors2y": 1 + }, + { + "package": "emittery", + "from": ">=1 <2", + "to": ">=2 <3", + "why": "1 breaking major in two years (2), 55,768,692 downloads a week", + "status": "todo", + "weeklyDownloads": 55768692, + "breakingMajors2y": 1 + } + ] +} diff --git a/scripts/packs-queue.ts b/scripts/packs-queue.ts new file mode 100644 index 0000000..fa5e10b --- /dev/null +++ b/scripts/packs-queue.ts @@ -0,0 +1,150 @@ +/** + * packs/queue.json: the next packages worth a pack, from public npm data only. + * + * pnpm packs:queue [--size=25] + * + * Candidates are the most downloaded packages the registry's search returns for a few broad + * queries, minus type stubs and build tooling (their upgrades break configuration, not the + * code that calls them), and minus what is pulled in transitively rather than imported: a + * package downloaded more than 50,000 times a week per package that depends on it + * (agent-base, content-type) is a dependency of dependencies. For each: weekly downloads + * (the search result) and the breaking majors published in the last two years (the + * packument's release dates), at most three counted (beyond that a package ships majors on a + * schedule, and a pack per major is not what it needs). The score is the product, roughly how + * many upgrades a pack would serve. `from → to` is the latest major and the + * one before it. Status: `verified` or `candidate` from the pack registry when a pack covers + * that upgrade; `in-progress` is kept from the existing file (someone said so in an issue); + * anything else is `todo`. + */ +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { packStatus, registeredPacks } from '@uptide/core'; + +const REGISTRY = 'https://registry.npmjs.org'; +const QUERIES = [ + 'keywords:typescript', + 'keywords:sdk', + 'keywords:api', + 'keywords:react', + 'keywords:validation', + 'keywords:orm', + 'keywords:database', + 'keywords:http', + 'keywords:ai', + 'keywords:framework', +]; +/** Build and test tooling: an upgrade changes its configuration, not the code that imports it. */ +const TOOLING = + /^(?:typescript|eslint|prettier|webpack|vite|vitest|jest|jest-.*|@angular-devkit\/.*|@angular\/cli|@vitejs\/.*|rollup|esbuild|tsup|turbo|@babel\/.*|@swc\/.*|@typescript-eslint\/.*|eslint-.*|postcss|tailwindcss|autoprefixer|nodemon|ts-node|tsx|lerna|nx|@nx\/.*|husky|lint-staged|rimraf|cross-env|dotenv|concurrently|npm|pnpm|yarn|typedoc|@types\/.*|tslib|core-js|regenerator-runtime|@biomejs\/biome|playwright|@playwright\/test|cypress|storybook|@storybook\/.*|sass|less|terser|webpack-cli|babel-.*|ts-jest|mocha|chai|sinon|nyc|c8|semver|debug|ms|chalk|picocolors|commander|yargs|minimist)$/; + +interface Entry { + package: string; + from: string; + to: string; + why: string; + status: 'todo' | 'in-progress' | 'candidate' | 'verified'; + weeklyDownloads: number; + breakingMajors2y: number; +} + +const size = Number(process.argv.find((a) => a.startsWith('--size='))?.slice(7) ?? 25); +const root = join(import.meta.dirname, '..'); +const file = join(root, 'packs', 'queue.json'); +const previous: Entry[] = existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')).queue : []; + +async function json(url: string): Promise { + for (let attempt = 0; ; attempt++) { + const res = await fetch(url, { headers: { accept: 'application/json' } }); + if (res.ok) return (await res.json()) as T; + if (attempt >= 3 || ![429, 500, 502, 503, 504].includes(res.status)) + throw new Error(`${url}: HTTP ${res.status}`); + await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)); + } +} + +/** Weekly downloads per dependent package above which a package is reached transitively. */ +const TRANSITIVE = 50_000; +const MAX_MAJORS = 3; +const weekly = new Map(); +for (const query of QUERIES) { + for (let from = 0; from < 500; from += 250) { + const page = await json<{ + objects: { package: { name: string }; downloads?: { weekly?: number } }[]; + }>( + `${REGISTRY}/-/v1/search?text=${encodeURIComponent(query)}&size=250&from=${from}&popularity=1.0&quality=0.0&maintenance=0.0`, + ); + for (const o of page.objects) { + const downloads = o.downloads?.weekly ?? 0; + const dependents = Number(o.dependents ?? 0); + if (TOOLING.test(o.package.name) || downloads > TRANSITIVE * Math.max(dependents, 1)) + continue; + weekly.set(o.package.name, downloads); + } + } +} +const top = [...weekly].sort((a, b) => b[1] - a[1]).slice(0, 300); + +const now = Date.now(); +const twoYears = 2 * 365 * 24 * 3600 * 1000; +const entries: Entry[] = []; +for (const [name, downloads] of top) { + const doc = await json<{ time?: Record; 'dist-tags'?: { latest?: string } }>( + `${REGISTRY}/${name.replace('/', '%2F')}`, + ).catch(() => undefined); + const latest = doc?.['dist-tags']?.latest; + if (!doc?.time || !latest) continue; + // The first stable release of each major, when it came out. + const firsts = new Map(); + for (const [version, at] of Object.entries(doc.time)) { + const m = /^(\d+)\.\d+\.\d+$/.exec(version); + if (!m) continue; + const major = Number(m[1]); + const time = Date.parse(at); + if (!firsts.has(major) || time < (firsts.get(major) as number)) firsts.set(major, time); + } + const latestMajor = Number(latest.split('.')[0]); + const recent = [...firsts].filter( + ([major, at]) => major >= 1 && major <= latestMajor && now - at <= twoYears, + ); + if (recent.length === 0 || latestMajor < 1) continue; + const from = `>=${latestMajor - 1} <${latestMajor}`; + const to = `>=${latestMajor} <${latestMajor + 1}`; + const pack = registeredPacks().find((e) => e.pack.name === name); + const covered = pack?.pack.supports(`${latestMajor - 1}.99.99`, `${latestMajor}.0.0`) + ? pack + : undefined; + const kept = previous.find((p) => p.package === name && p.to === to)?.status; + const majors = recent.map(([m]) => m).sort((a, b) => a - b); + entries.push({ + package: name, + from, + to, + why: `${recent.length} breaking major${recent.length === 1 ? '' : 's'} in two years (${majors.join(', ')}), ${downloads.toLocaleString('en-US')} downloads a week`, + status: covered ? packStatus(covered) : kept === 'in-progress' ? 'in-progress' : 'todo', + weeklyDownloads: downloads, + breakingMajors2y: Math.min(recent.length, MAX_MAJORS), + }); +} +const queue = entries + .sort( + (a, b) => + b.weeklyDownloads * b.breakingMajors2y - a.weeklyDownloads * a.breakingMajors2y || + a.package.localeCompare(b.package), + ) + .slice(0, size); +writeFileSync( + file, + `${JSON.stringify( + { + $comment: + 'Generated by `pnpm packs:queue` from public npm data: weekly downloads (registry search) times breaking majors in the last two years (registry release dates). Pick one, say so in a pack-request issue, and follow CONTRIBUTING.md, "Write a pack".', + generated: new Date().toISOString().slice(0, 10), + queue, + }, + null, + 2, + )}\n`, +); +console.log(`${file}: ${queue.length} packages`); +for (const e of queue) + console.log(` ${e.status.padEnd(11)} ${e.package} ${e.from} → ${e.to} ${e.why}`); From ad301b1ffa5ee958105c962295f4fbebb4754437 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 07:22:34 +0000 Subject: [PATCH 4/9] feat(packs): draft ground truth from the compiler and the upgrade commit pnpm packs:truth lists, for one public repository, the diagnostics its own TypeScript reports with the target linked in and not without it (errors and deprecations), and the lines its upgrade commit changed. Ground truth can name a project directory inside the repository; an install whose lockfile was already out of sync falls back to an unfrozen install, scripts still off. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01B2T8GmEVnZwfNrugajEMqi Signed-off-by: Claude --- package.json | 3 +- packages/core/src/packs/contract.ts | 2 + packages/core/src/packs/tooling/pack-test.ts | 12 +- packages/core/src/packs/tooling/truth.ts | 31 ++-- scripts/packs-truth-compiler.ts | 82 +++++++++ scripts/packs-truth.ts | 165 +++++++++++++++++++ 6 files changed, 281 insertions(+), 14 deletions(-) create mode 100644 scripts/packs-truth-compiler.ts create mode 100644 scripts/packs-truth.ts diff --git a/package.json b/package.json index c2d6931..23c5321 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,8 @@ "public:check": "node scripts/public-tree.mjs . --require-denylist", "uptide": "node --conditions=source --import tsx packages/cli/src/index.ts", "packs:fetch": "node --conditions=source --import tsx scripts/packs-fetch.ts", - "packs:queue": "node --conditions=source --import tsx scripts/packs-queue.ts" + "packs:queue": "node --conditions=source --import tsx scripts/packs-queue.ts", + "packs:truth": "node --conditions=source --import tsx scripts/packs-truth.ts" }, "devDependencies": { "@biomejs/biome": "^2.5.14", diff --git a/packages/core/src/packs/contract.ts b/packages/core/src/packs/contract.ts index 4a1a904..6acb0b5 100644 --- a/packages/core/src/packs/contract.ts +++ b/packages/core/src/packs/contract.ts @@ -98,6 +98,8 @@ export interface GroundTruthRepo { commit?: string; /** A repository inside this one, relative to its root, instead of `repo` and `commit`. */ fixture?: string; + /** The project inside the repository, when it is not at its root (`frontend`). */ + directory?: string; /** Where the expected findings come from: the commit or pull request that made the upgrade. */ migration?: string; /** The exact version installed at `commit` and the one to check against. */ diff --git a/packages/core/src/packs/tooling/pack-test.ts b/packages/core/src/packs/tooling/pack-test.ts index 910fc96..6cbd748 100644 --- a/packages/core/src/packs/tooling/pack-test.ts +++ b/packages/core/src/packs/tooling/pack-test.ts @@ -161,10 +161,18 @@ async function scoreRepo( try { const dir = entry.fixture ? await ensureFixture(root, entry.fixture, options) - : await ensureRepo({ repo: entry.repo as string, commit: entry.commit as string }, options); + : await ensureRepo( + { + repo: entry.repo as string, + commit: entry.commit as string, + ...(entry.directory ? { directory: entry.directory } : {}), + }, + options, + ); + const project = entry.directory ? join(dir, entry.directory) : dir; options.log?.(`checking ${pack.name} ${entry.from} → ${entry.to} in ${label(entry)}`); const report = await check({ - cwd: dir, + cwd: project, only: [pack.name], targets: { [pack.name]: entry.to }, packs: [pack], diff --git a/packages/core/src/packs/tooling/truth.ts b/packages/core/src/packs/tooling/truth.ts index ada7d16..f1c1c09 100644 --- a/packages/core/src/packs/tooling/truth.ts +++ b/packages/core/src/packs/tooling/truth.ts @@ -20,6 +20,8 @@ export function truthCacheDir(env: NodeJS.ProcessEnv = process.env): string { export interface PinnedRepo { repo: string; commit: string; + /** Where to install, when the project is not at the repository root. */ + directory?: string; } const SHA = /^[0-9a-f]{40}$/; @@ -41,6 +43,8 @@ export function truthProblems(truth: GroundTruth, packageName: string): string[] if (!r.commit || !SHA.test(r.commit)) problems.push(`${where}: commit must be a full 40-character SHA`); } else if (r.repo || r.commit) problems.push(`${where}: a fixture has no repo or commit`); + if (r.directory && (r.directory.startsWith('/') || /(^|\/)\.\.(\/|$)/.test(r.directory))) + problems.push(`${where}: directory is relative and inside the repository`); if (!r.from || !r.to) problems.push(`${where}: from and to are exact versions`); for (const f of r.findings ?? []) if (!f.file || !Number.isInteger(f.line) || f.line < 1 || !f.rule) @@ -100,8 +104,8 @@ async function installDependencies( interface Marker { repo: string; commit: string; - installed: boolean; - manager?: string; + /** The directories installed so far (`.` for the root). */ + installed: string[]; } const MARKER = '.uptide-ground-truth.json'; @@ -135,7 +139,7 @@ export async function ensureRepo(entry: PinnedRepo, options: EnsureOptions = {}) const dir = cachedRepoDir(entry, cacheDir); const install = options.install ?? true; const marker = readMarker(dir); - if (marker && (marker.installed || !install)) return dir; + if (marker && (!install || marker.installed.includes(entry.directory ?? '.'))) return dir; if (options.offline) throw new UptideError( 'GROUND_TRUTH_NOT_CACHED', @@ -165,18 +169,23 @@ export async function ensureRepo(entry: PinnedRepo, options: EnsureOptions = {}) ); } } - let installed = false; - let manager: string | undefined; - let frozen: boolean | undefined; + const installed = [...(marker?.installed ?? [])]; if (install) { - const result = await installDependencies(dir, entry.repo, options); - manager = result.manager; - frozen = result.frozen; - installed = true; + const project = entry.directory ? join(dir, entry.directory) : dir; + if (entry.directory && (/(^|\/)\.\.(\/|$)/.test(entry.directory) || !existsSync(project))) + throw new UptideError( + 'INVALID_GROUND_TRUTH', + `no directory ${entry.directory} in ${entry.repo}`, + ); + const result = await installDependencies(project, entry.repo, options); + installed.push(entry.directory ?? '.'); + options.log?.( + `installed with ${result.manager}${result.frozen ? '' : ', lockfile not frozen'}`, + ); } writeFileSync( join(dir, MARKER), - `${JSON.stringify({ repo: entry.repo, commit: entry.commit, installed, ...(manager ? { manager, frozen } : {}) })}\n`, + `${JSON.stringify({ repo: entry.repo, commit: entry.commit, installed })}\n`, ); return dir; } diff --git a/scripts/packs-truth-compiler.ts b/scripts/packs-truth-compiler.ts new file mode 100644 index 0000000..38368d5 --- /dev/null +++ b/scripts/packs-truth-compiler.ts @@ -0,0 +1,82 @@ +import { createRequire } from 'node:module'; +import { dirname, join, relative } from 'node:path'; + +export interface Diagnostic { + file: string; + line: number; + code: number; + message: string; +} + +/** Deprecated declarations, as the language service reports them to an editor. */ +const DEPRECATED = new Set([6385, 6387]); + +/** + * Errors and deprecation suggestions of every program a repository's tsconfig files describe, + * computed by the repository's own TypeScript (the one it installed), project-relative. + */ +export function compilerDiagnostics( + project: string, + configs: string[], +): { compiler: string; diagnostics: Diagnostic[] } { + const require = createRequire(join(project, 'package.json')); + let ts = require( + require.resolve('typescript', { + paths: [project, ...configs.map((c) => join(project, dirname(c)))], + }), + ) as typeof import('typescript'); + let compiler = `typescript ${ts.version}, the repository's`; + // TypeScript 7 has no JavaScript API: the one this checkout builds with stands in, said so. + if (typeof ts.getParsedCommandLineOfConfigFile !== 'function') { + const own = createRequire(import.meta.url)('typescript') as typeof import('typescript'); + compiler = `typescript ${own.version}, this checkout's (the repository's ${(ts as { version?: string }).version ?? 'own'} has no JavaScript API)`; + ts = own; + } + const out: Diagnostic[] = []; + for (const config of configs) { + const file = join(project, config); + const parsed = ts.getParsedCommandLineOfConfigFile( + file, + {}, + { ...ts.sys, onUnRecoverableConfigFileDiagnostic: () => {} }, + ); + if (!parsed) continue; + const service = ts.createLanguageService({ + getScriptFileNames: () => parsed.fileNames, + getScriptVersion: () => '1', + getScriptSnapshot: (f) => + ts.sys.fileExists(f) ? ts.ScriptSnapshot.fromString(ts.sys.readFile(f) ?? '') : undefined, + getCurrentDirectory: () => dirname(file), + getCompilationSettings: () => parsed.options, + getDefaultLibFileName: (o) => ts.getDefaultLibFilePath(o), + fileExists: ts.sys.fileExists, + readFile: ts.sys.readFile, + readDirectory: ts.sys.readDirectory, + directoryExists: ts.sys.directoryExists, + getDirectories: ts.sys.getDirectories, + realpath: ts.sys.realpath, + }); + for (const source of parsed.fileNames) { + if (source.includes('/node_modules/')) continue; + const diagnostics = [ + ...service.getSyntacticDiagnostics(source), + ...service.getSemanticDiagnostics(source), + ...service.getSuggestionDiagnostics(source).filter((d) => DEPRECATED.has(d.code)), + ]; + for (const d of diagnostics) { + if (!d.file || d.start === undefined) continue; + if (d.category !== ts.DiagnosticCategory.Error && !DEPRECATED.has(d.code)) continue; + out.push({ + file: relative(project, d.file.fileName), + line: d.file.getLineAndCharacterOfPosition(d.start).line + 1, + code: d.code, + message: ts.flattenDiagnosticMessageText(d.messageText, ' '), + }); + } + } + } + return { + compiler, + diagnostics: out.sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line), + }; +} diff --git a/scripts/packs-truth.ts b/scripts/packs-truth.ts new file mode 100644 index 0000000..aa828c9 --- /dev/null +++ b/scripts/packs-truth.ts @@ -0,0 +1,165 @@ +/** + * A draft of the expected findings for one ground-truth repository, from two sources that are + * not Uptide: the repository's own upgrade commit, and its own TypeScript compiler. + * + * pnpm packs:truth --to [--directory ] + * + * - changed: every line of `` the upgrade commit changed or removed (`git diff -U0`, + * the old side), in source files. Pure insertions have no line in `` and are listed + * apart. + * - compiler: the diagnostics the repository's own TypeScript reports with the target version + * swapped in and not without it, errors and deprecations (6385, 6387), keyed on file, code + * and message. The target is linked where the repository has the package installed, as an + * install of it would put it. + * + * It prints JSON for docs/packs.md, "Ground truth": you read each site, drop what is not about + * the package, and give every kept site its rule. Nothing here comes from `check`. + */ +import { execFileSync } from 'node:child_process'; +import { + existsSync, + lstatSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + symlinkSync, +} from 'node:fs'; +import { join, relative } from 'node:path'; +import { createNpmFetcher, ensureRepo } from '@uptide/core'; +import { compilerDiagnostics, type Diagnostic } from './packs-truth-compiler.ts'; + +const args = process.argv.slice(2); +const flag = (name: string): string | undefined => { + const i = args.indexOf(name); + return i >= 0 ? args[i + 1] : undefined; +}; +const positional = args.filter((a, i) => !a.startsWith('--') && !args[i - 1]?.startsWith('--')); +const [pkg, repo, commit, upgrade] = positional; +const to = flag('--to'); +const directory = flag('--directory'); +if (!pkg || !repo || !commit || !upgrade || !to) { + console.error( + 'usage: pnpm packs:truth --to [--directory ]', + ); + process.exit(2); +} + +const root = await ensureRepo( + { repo, commit, ...(directory ? { directory } : {}) }, + { log: (line) => console.error(line) }, +); +const project = directory ? join(root, directory) : root; + +// The upgrade commit, next to the pinned one: one more shallow fetch. +const git = (...a: string[]) => + execFileSync('git', ['-c', 'core.hooksPath=/dev/null', ...a], { + cwd: root, + encoding: 'utf8', + maxBuffer: 256 * 1024 * 1024, + }); +git('fetch', '-q', '--depth', '1', '--no-tags', 'origin', upgrade); +const SOURCE = /\.[cm]?[jt]sx?$/; +const changed: { file: string; line: number; text: string }[] = []; +const inserted: { file: string; after: number; text: string }[] = []; +let file = ''; +let oldLine = 0; +let newRun = false; +for (const line of git('diff', '-U0', '--no-color', '--no-ext-diff', commit, upgrade).split('\n')) { + if (line.startsWith('--- ')) continue; + if (line.startsWith('+++ ')) { + file = line.slice(4).replace(/^b\//, ''); + continue; + } + const hunk = /^@@ -(\d+)(?:,(\d+))? \+\d+(?:,\d+)? @@/.exec(line); + if (hunk) { + oldLine = Number(hunk[1]); + newRun = hunk[2] === '0'; + if (newRun) oldLine += 1; + continue; + } + if (!SOURCE.test(file) || (directory && !file.startsWith(`${directory}/`))) continue; + const where = directory ? relative(directory, file) : file; + if (line.startsWith('-')) + changed.push({ file: where, line: oldLine++, text: line.slice(1).trim() }); + else if (line.startsWith('+') && newRun) + inserted.push({ file: where, after: oldLine - 1, text: line.slice(1).trim() }); +} + +// Where the repository has the package: the root's node_modules and each workspace's. +function installs(dir: string, depth = 0): string[] { + const found: string[] = []; + const own = join(dir, 'node_modules', ...pkg.split('/')); + if (existsSync(own)) found.push(relative(project, own)); + if (depth >= 3) return found; + for (const entry of readdirSync(dir, { withFileTypes: true })) + if (entry.isDirectory() && !['node_modules', '.git', 'dist', 'build'].includes(entry.name)) + found.push(...installs(join(dir, entry.name), depth + 1)); + return found; +} +function tsconfigs(dir: string, depth = 0): string[] { + const found = existsSync(join(dir, 'tsconfig.json')) + ? [relative(project, join(dir, 'tsconfig.json'))] + : []; + if (depth >= 3) return found; + for (const entry of readdirSync(dir, { withFileTypes: true })) + if (entry.isDirectory() && !['node_modules', '.git', 'dist', 'build'].includes(entry.name)) + found.push(...tsconfigs(join(dir, entry.name), depth + 1)); + return found; +} +const links = installs(project); +const configs = tsconfigs(project); +const fetcher = createNpmFetcher(); +const target = await fetcher.fetch(pkg, to); + +const { compiler: compilerName, diagnostics: baseline } = compilerDiagnostics(project, configs); +for (const link of links) { + const at = join(project, link); + renameSync(at, `${at}.uptide-orig`); + symlinkSync(target.dir, at); +} +let after: Diagnostic[]; +try { + after = compilerDiagnostics(project, configs).diagnostics; +} finally { + for (const link of links) { + const at = join(project, link); + if (lstatSync(at).isSymbolicLink()) rmSync(at); + renameSync(`${at}.uptide-orig`, at); + } +} +const key = (d: Diagnostic) => `${d.file} ${d.code} ${d.message}`; +const pool = new Map(); +for (const d of baseline) pool.set(key(d), (pool.get(key(d)) ?? 0) + 1); +const compiler = after.filter((d) => { + const left = pool.get(key(d)) ?? 0; + if (left > 0) { + pool.set(key(d), left - 1); + return false; + } + return true; +}); + +console.log( + JSON.stringify( + { + repo, + commit, + upgrade, + ...(directory ? { directory } : {}), + to, + links, + compiledWith: compilerName, + tsconfigs: configs, + compiler: compiler.map((d) => ({ + ...d, + message: d.message.slice(0, 200), + text: (readFileSync(join(project, d.file), 'utf8').split('\n')[d.line - 1] ?? '').trim(), + })), + changed, + inserted, + }, + null, + 2, + ), +); From ad78550b6d430d526c256e2cbb86de159425d2fc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 07:22:34 +0000 Subject: [PATCH 5/9] test(packs): zod and stripe verified against public repositories zod: AwesomeIntune, PATH and band-sdk-typescript at the commit before their zod 4 upgrade; stripe: unkey and saas-starter before their stripe-node majors; both also scored on the storefront fixture. No false positive among breaking findings (zod 57 sites, recall 76%; stripe 18 sites, recall 90%). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01B2T8GmEVnZwfNrugajEMqi Signed-off-by: Claude --- CONTRIBUTING.md | 10 +- docs/packs.md | 36 +- .../core/src/packs/stripe/ground-truth.json | 119 +++- .../core/src/packs/stripe/verification.json | 8 +- packages/core/src/packs/zod/ground-truth.json | 620 +++++++++++++++++- packages/core/src/packs/zod/verification.json | 10 +- packs/queue.json | 72 +- 7 files changed, 820 insertions(+), 55 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8f9977c..3f4d7e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -123,9 +123,13 @@ open a **Pack request** issue, and say you are writing it. (`// @uptide `, `keep` for one it must leave alone) and `after.ts`, what the rules must produce. `pnpm uptide pack test --fixtures-only` runs them in seconds. 5. **Find ground truth.** Two public repositories that made this upgrade in a commit you - can point at. Pin the commit before it; the expected findings are the sites the upgrade - commit changed for this package (`git diff -U0`, old line numbers), - each with its rule (`generic` when no rule of the pack covers it). (15 minutes) + can point at. Pin the commit before it, and draft the expected findings from their own + compiler and their own upgrade commit: + ```sh + pnpm packs:truth owner/name --to + ``` + Read each site, drop what is not about the package, and give each its rule (`generic` + when no rule of the pack covers it). (15 minutes) 6. **Score it.** ```sh pnpm uptide pack test # fetches the repositories once, scripts off diff --git a/docs/packs.md b/docs/packs.md index ddabab0..6f4ef24 100644 --- a/docs/packs.md +++ b/docs/packs.md @@ -134,8 +134,9 @@ Runtime never downloads instructions: they are checked in, with the date they we "repo": "owner/name", "commit": "", "migration": "https://github.com/owner/name/commit/", + "directory": "web", "from": "3.25.76", - "to": "4.1.12", + "to": "4.6.5", "why": "what this repository exercises", "findings": [{ "file": "src/schema.ts", "line": 12, "rule": "error-params" }] } @@ -148,16 +149,33 @@ findings the pack must report there: repository-relative file, 1-based line, and or note id. `from` is the version the lockfile has at that commit, and `to` the version the repository upgraded to. -Where the expected findings come from: +Where the expected findings come from: two sources that are not Uptide. -1. the sites the repository's own upgrade commit changed for this dependency (`git diff - -U0`, the old side's line numbers), and -2. the errors the compiler reports at the target that the upgrade commit fixed, +1. **The compiler.** The repository's own TypeScript, run on every `tsconfig.json`, with the + target version linked where the package is installed and without it. Every diagnostic + new at the target is a site: errors, and deprecations (6385, 6387, what an editor strikes + through). A repository on TypeScript 7, which has no JavaScript API, is compiled with the + TypeScript of this checkout, and the draft says so. +2. **The upgrade commit.** The lines it changed or removed (`git diff -U0 `, + the old side). One site per construct: a run of changed lines is one site, at its first + line of code, and a run that a diagnostic falls inside (or just after) is that + diagnostic's site. Comment-only lines, pure insertions (no line at ``), and + changes for another package are not sites. -each read and kept only when it is about this dependency. A site that `check` reports and -that nobody had to change is a false positive, and it stays one: the expected findings -never come from Uptide's own output. A site the pack has no rule for is expected under -`generic`, which is what `pack test` calls a site the plan groups under no rule of the pack. +`pnpm packs:truth` drafts both for one repository: + +```sh +pnpm packs:truth zod owner/name --to 4.6.5 [--directory web] +``` + +It prints the compiler's new diagnostics with each line's text, and the changed and inserted +lines. You read each one, drop what is not about the package, and give every kept site its +rule, from its code and text. A site the pack has no rule for is expected under `generic`. +The expected findings never come from Uptide's output: a site `check` reports that neither +source has is a false positive, and it stays one. + +`directory` is the project inside the repository when it is not at the root; `from` is the +version its lockfile has at `commit`, and `pack test` says so when it is not. `fixture` entries (`"fixture": "fixtures/repos/storefront"`) are repositories in this tree. They are scored like the others and never count toward `verified`. diff --git a/packages/core/src/packs/stripe/ground-truth.json b/packages/core/src/packs/stripe/ground-truth.json index 485ac0c..5a7804c 100644 --- a/packages/core/src/packs/stripe/ground-truth.json +++ b/packages/core/src/packs/stripe/ground-truth.json @@ -28,7 +28,124 @@ "rule": "subscription-period" } ] + }, + { + "repo": "unkeyed/unkey", + "commit": "37044fc57fe08c917e92b323933baf322d6df1c6", + "migration": "https://github.com/unkeyed/unkey/commit/234357b0226c2d2dc7881e096d1ff216ea02ccf8", + "directory": "web", + "from": "14.23.0", + "to": "22.2.0", + "why": "pnpm monorepo dashboard: two apiVersion pins, Invoice.subscription and payment_intent moved, the subscription period on items, Stripe.errors used as a type", + "findings": [ + { + "file": "apps/dashboard/app/api/webhooks/stripe/route.ts", + "line": 88, + "rule": "api-version", + "note": "compiler TS2322" + }, + { + "file": "apps/dashboard/lib/stripe.ts", + "line": 26, + "rule": "api-version", + "note": "compiler TS2322" + }, + { + "file": "apps/dashboard/lib/stripe/paymentUtils.ts", + "line": 253, + "rule": "sdk-surface", + "note": "compiler TS2339" + }, + { + "file": "apps/dashboard/lib/stripe/paymentUtils.ts", + "line": 258, + "rule": "sdk-surface", + "note": "compiler TS2339" + }, + { + "file": "apps/dashboard/lib/stripe/paymentUtils.ts", + "line": 264, + "rule": "sdk-surface", + "note": "compiler TS2339" + }, + { + "file": "apps/dashboard/lib/stripe/paymentUtils.ts", + "line": 281, + "rule": "subscription-period", + "note": "compiler TS2339" + }, + { + "file": "apps/dashboard/lib/stripe/paymentUtils.ts", + "line": 287, + "rule": "subscription-period", + "note": "changed by the upgrade" + }, + { + "file": "apps/dashboard/lib/stripe/paymentUtils.ts", + "line": 363, + "rule": "sdk-surface", + "note": "compiler TS2339" + }, + { + "file": "apps/dashboard/lib/stripe/paymentUtils.ts", + "line": 365, + "rule": "sdk-surface", + "note": "compiler TS2339" + }, + { + "file": "apps/dashboard/lib/stripe/paymentUtils.ts", + "line": 366, + "rule": "sdk-surface", + "note": "compiler TS2339" + }, + { + "file": "apps/dashboard/lib/stripe/paymentUtils.ts", + "line": 367, + "rule": "sdk-surface", + "note": "compiler TS2339" + }, + { + "file": "apps/dashboard/lib/stripe/paymentUtils.ts", + "line": 450, + "rule": "sdk-surface", + "note": "compiler TS2339" + }, + { + "file": "apps/dashboard/lib/stripe/subscriptionUtils.ts", + "line": 118, + "rule": "sdk-surface", + "note": "changed by the upgrade" + }, + { + "file": "apps/dashboard/lib/trpc/routers/utils/stripe.ts", + "line": 49, + "rule": "sdk-surface", + "note": "compiler TS2749" + }, + { + "file": "tools/migrate/migrate_subscription.ts", + "line": 3, + "rule": "sdk-surface", + "note": "compiler TS2866" + } + ] + }, + { + "repo": "nextjs/saas-starter", + "commit": "3e5fd2bf626c4ee73c06dcc9e1aa503b8904a37c", + "migration": "https://github.com/nextjs/saas-starter/commit/e88fb409e09a23628331f84bcd485baf68d693b5", + "from": "17.7.0", + "to": "18.0.0", + "why": "Next.js starter across the basil boundary (2025-03-31): the apiVersion pin; the rest of the upgrade commit is an unrelated UI refactor, left out", + "findings": [ + { + "file": "lib/payments/stripe.ts", + "line": 11, + "rule": "api-version", + "note": "compiler TS2322" + } + ] } ], - "$comment": "Ground truth for the stripe pack. Public repositories at the commit before a stripe-node major upgrade, each with every site the real upgrade changed for stripe or the compiler rejected (docs/packs.md, \"Ground truth\"). The storefront fixture is synthetic: scored, never counted toward verified; its sites are the tsc errors of fixtures/truth/storefront.json." + "$comment": "Ground truth for the stripe pack (docs/packs.md, \"Ground truth\"). Each public repository is pinned at the commit before its own stripe-node major upgrade. Expected findings come from two sources that are not Uptide, drafted by pnpm packs:truth and read by hand: the errors the repository's TypeScript reports with the target SDK linked in and not without it, and the lines the upgrade commit changed for stripe (one site per construct; comments and unrelated changes left out). Rules are assigned from each site's code and text. The storefront fixture is synthetic: scored, never counted toward verified; its sites are the tsc errors of fixtures/truth/storefront.json." } diff --git a/packages/core/src/packs/stripe/verification.json b/packages/core/src/packs/stripe/verification.json index d596ce8..9036546 100644 --- a/packages/core/src/packs/stripe/verification.json +++ b/packages/core/src/packs/stripe/verification.json @@ -1,6 +1,10 @@ { "status": "verified", - "truth": "", + "truth": "sha256-76142ce85f863173121202c804005de14fb0beacc54f40c5dbe560865ed7ef15", "repos": 2, - "breaking": { "precision": 1, "recall": 1, "falsePositives": 0 } + "breaking": { + "precision": 1, + "recall": 0.9, + "falsePositives": 0 + } } diff --git a/packages/core/src/packs/zod/ground-truth.json b/packages/core/src/packs/zod/ground-truth.json index 7d7179f..0e00e84 100644 --- a/packages/core/src/packs/zod/ground-truth.json +++ b/packages/core/src/packs/zod/ground-truth.json @@ -198,7 +198,625 @@ "rule": "error-params" } ] + }, + { + "repo": "ugurkocde/AwesomeIntune", + "commit": "dcf38603fd55cea72737e17274d5de03cece341e", + "migration": "https://github.com/ugurkocde/AwesomeIntune/commit/c980a77e4863578f3a26281d9c20505bc43b1e01", + "from": "3.25.76", + "to": "4.6.5", + "why": "Next.js app (npm): ZodError.errors, errorMap on z.literal, string formats in schemas and env.js", + "findings": [ + { + "file": "src/app/api/ideas/route.ts", + "line": 201, + "rule": "generic", + "note": "compiler TS6387" + }, + { + "file": "src/app/api/submit-tool/route.ts", + "line": 17, + "rule": "generic", + "note": "compiler TS6387" + }, + { + "file": "src/app/api/subscribe/route.ts", + "line": 10, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "src/app/api/subscribe/route.ts", + "line": 23, + "rule": "generic", + "note": "compiler TS2339" + }, + { + "file": "src/app/api/v1/keys/route.ts", + "line": 30, + "rule": "generic", + "note": "compiler TS2339" + }, + { + "file": "src/env.js", + "line": 12, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "src/env.js", + "line": 25, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "src/lib/schemas/api-key.ts", + "line": 8, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "src/lib/schemas/tool-submission.ts", + "line": 35, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "src/lib/schemas/tool-submission.ts", + "line": 44, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "src/lib/schemas/tool-submission.ts", + "line": 53, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "src/lib/schemas/tool-submission.ts", + "line": 80, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "src/lib/schemas/tool-submission.ts", + "line": 85, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "src/lib/schemas/tool-submission.ts", + "line": 86, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "src/lib/schemas/tool-submission.ts", + "line": 99, + "rule": "error-params", + "note": "compiler TS2769" + } + ] + }, + { + "repo": "howardyang2009/PATH", + "commit": "bb171417d911f373c8df28ab7a1e0698b62e0906", + "migration": "https://github.com/howardyang2009/PATH/commit/bf66cc6c189d7e61f2c810680d3d956d581351fe", + "from": "3.25.76", + "to": "4.6.5", + "why": "pnpm workspace of four packages: single-argument z.record, ZodDiscriminatedUnionOption, a ZodErrorMap with ctx, recursive lazy schemas, _def.typeName reads", + "findings": [ + { + "file": "packages/designer/src/open-workflow.ts", + "line": 148, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/engine/src/run-workflow.ts", + "line": 1208, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/engine/step-plugins/person-activity/index.ts", + "line": 5, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/engine/step-plugins/prompt/index.ts", + "line": 41, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/schema/src/conditions.ts", + "line": 9, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/conditions.ts", + "line": 20, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/config.ts", + "line": 63, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/schema/src/config.ts", + "line": 73, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/config.ts", + "line": 83, + "rule": "types", + "note": "compiler TS2322" + }, + { + "file": "packages/schema/src/config.ts", + "line": 102, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/schema/src/interpolation.ts", + "line": 111, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/interpolation.ts", + "line": 117, + "rule": "types", + "note": "compiler TS2322" + }, + { + "file": "packages/schema/src/interpolation.ts", + "line": 124, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/schema/src/nodes.ts", + "line": 1, + "rule": "types", + "note": "compiler TS2724" + }, + { + "file": "packages/schema/src/nodes.ts", + "line": 22, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/schema/src/nodes.ts", + "line": 59, + "rule": "types", + "note": "changed by the upgrade" + }, + { + "file": "packages/schema/src/nodes.ts", + "line": 206, + "rule": "types", + "note": "changed by the upgrade" + }, + { + "file": "packages/schema/src/nodes.ts", + "line": 229, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/nodes.ts", + "line": 255, + "rule": "error-params", + "note": "compiler TS2322" + }, + { + "file": "packages/schema/src/nodes.ts", + "line": 256, + "rule": "generic", + "note": "compiler TS2339" + }, + { + "file": "packages/schema/src/nodes.ts", + "line": 260, + "rule": "generic", + "note": "changed by the upgrade" + }, + { + "file": "packages/schema/src/nodes.ts", + "line": 292, + "rule": "error-params", + "note": "compiler TS2353" + }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 1, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 60, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 77, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 79, + "rule": "types", + "note": "compiler TS2352" + }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 83, + "rule": "generic", + "note": "changed by the upgrade" + }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 84, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 87, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 91, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 94, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 104, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/workflow-file.ts", + "line": 25, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/schema/src/workflow-file.ts", + "line": 176, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/workflow-file.ts", + "line": 185, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/workflow-file.ts", + "line": 193, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/schema/src/workflow-file.ts", + "line": 210, + "rule": "types", + "note": "compiler TS2345" + }, + { + "file": "packages/schema/test/builtin-registry.ts", + "line": 22, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/schema/test/wire-step-plugins.test.ts", + "line": 27, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/server/src/routes/post-runs.ts", + "line": 12, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/server/src/routes/post-runs.ts", + "line": 55, + "rule": "generic", + "note": "compiler TS2345" + }, + { + "file": "packages/server/src/routes/post-runs.ts", + "line": 82, + "rule": "generic", + "note": "compiler TS2322" + }, + { + "file": "packages/server/src/routes/put-workflow.ts", + "line": 22, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/server/src/routes/resume-run.ts", + "line": 52, + "rule": "generic", + "note": "compiler TS2345" + }, + { + "file": "packages/server/src/routes/resume-run.ts", + "line": 127, + "rule": "generic", + "note": "compiler TS2322" + }, + { + "file": "scripts/test/builtin-registry.ts", + "line": 23, + "rule": "generic", + "note": "compiler TS2554" + } + ] + }, + { + "repo": "band-ai/band-sdk-typescript", + "commit": "5d9a66a1dde4989fbc0608200f291f0ca8194052", + "migration": "https://github.com/band-ai/band-sdk-typescript/commit/c5ef5dd0fabf836104a5b2c13a2e29abc6a97127", + "from": "3.25.76", + "to": "4.6.5", + "why": "pnpm workspace SDK: AnyZodObject, single-argument z.record, ZodTypeAny and ZodIssue, string formats", + "findings": [ + { + "file": "packages/sdk/src/adapters/claude-sdk/ClaudeSDKAdapter.ts", + "line": 92, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/integrations/linear/tools.ts", + "line": 53, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/integrations/linear/tools.ts", + "line": 175, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/integrations/linear/tools.ts", + "line": 289, + "rule": "types", + "note": "compiler TS2724" + }, + { + "file": "packages/sdk/src/integrations/linear/tools.ts", + "line": 290, + "rule": "types", + "note": "compiler TS2724" + }, + { + "file": "packages/sdk/src/integrations/linear/tools.ts", + "line": 612, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/integrations/linear/tools.ts", + "line": 617, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/integrations/linear/tools.ts", + "line": 676, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/integrations/linear/tools.ts", + "line": 700, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/integrations/linear/tools.ts", + "line": 704, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/integrations/linear/tools.ts", + "line": 705, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/integrations/linear/tools.ts", + "line": 706, + "rule": "string-format", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/mcp/sdk.ts", + "line": 359, + "rule": "types", + "note": "compiler TS2345" + }, + { + "file": "packages/sdk/src/mcp/server.ts", + "line": 311, + "rule": "types", + "note": "compiler TS2322" + }, + { + "file": "packages/sdk/src/mcp/sse.ts", + "line": 239, + "rule": "types", + "note": "compiler TS2322" + }, + { + "file": "packages/sdk/src/mcp/stdio.ts", + "line": 93, + "rule": "types", + "note": "compiler TS2322" + }, + { + "file": "packages/sdk/src/mcp/zod.ts", + "line": 5, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/mcp/zod.ts", + "line": 6, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/mcp/zod.ts", + "line": 19, + "rule": "types", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/disconnectReason.ts", + "line": 109, + "rule": "generic", + "note": "compiler TS2554" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 8, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 12, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 25, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 31, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 44, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 52, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 59, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 63, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 67, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 76, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 81, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 91, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/platform/streaming/payloadSchemas.ts", + "line": 95, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/runtime/tools/customTools.ts", + "line": 1, + "rule": "generic", + "note": "compiler TS6385" + }, + { + "file": "packages/sdk/src/runtime/tools/customTools.ts", + "line": 5, + "rule": "types", + "note": "compiler TS2724" + }, + { + "file": "packages/sdk/src/runtime/tools/customTools.ts", + "line": 78, + "rule": "types", + "note": "compiler TS2724" + }, + { + "file": "packages/sdk/src/runtime/tools/customTools.ts", + "line": 146, + "rule": "generic", + "note": "compiler TS6385" + } + ] } ], - "$comment": "Ground truth for the zod pack. Public repositories at the commit before they upgraded zod 3 to 4, each with every site the real upgrade changed for zod or the compiler rejected (docs/packs.md, \"Ground truth\"). The storefront fixture is synthetic: scored, never counted toward verified; its breaking sites are the tsc errors of fixtures/truth/storefront.json, its deprecated ones the zod 3 APIs zod 4 deprecates (string formats, ZodIssue, flatten)." + "$comment": "Ground truth for the zod pack (docs/packs.md, \"Ground truth\"). Each public repository is pinned at the commit before its own zod 3 to 4 upgrade. Expected findings come from two sources that are not Uptide, drafted by pnpm packs:truth and read by hand: the diagnostics the repository's TypeScript reports with zod 4.6.5 linked in and not without it (errors, and deprecations 6385/6387), and the lines the upgrade commit changed for zod (one site per construct; comments and other packages' changes left out). Rules are assigned from each site's code and text. The storefront fixture is synthetic: scored, never counted toward verified; its breaking sites are the tsc errors of fixtures/truth/storefront.json." } diff --git a/packages/core/src/packs/zod/verification.json b/packages/core/src/packs/zod/verification.json index d596ce8..fb5186f 100644 --- a/packages/core/src/packs/zod/verification.json +++ b/packages/core/src/packs/zod/verification.json @@ -1,6 +1,10 @@ { "status": "verified", - "truth": "", - "repos": 2, - "breaking": { "precision": 1, "recall": 1, "falsePositives": 0 } + "truth": "sha256-ce97dc27050c267807a0765d9a472078fbcef84eb65877d5ecc1597be45f1957", + "repos": 3, + "breaking": { + "precision": 1, + "recall": 0.76, + "falsePositives": 0 + } } diff --git a/packs/queue.json b/packs/queue.json index b8c8a74..f476303 100644 --- a/packs/queue.json +++ b/packs/queue.json @@ -29,15 +29,6 @@ "weeklyDownloads": 119280995, "breakingMajors2y": 2 }, - { - "package": "@vitejs/plugin-react", - "from": ">=5 <6", - "to": ">=6 <7", - "why": "2 breaking majors in two years (5, 6), 116,919,148 downloads a week", - "status": "todo", - "weeklyDownloads": 116919148, - "breakingMajors2y": 2 - }, { "package": "react", "from": ">=18 <19", @@ -137,15 +128,6 @@ "weeklyDownloads": 49967370, "breakingMajors2y": 2 }, - { - "package": "@angular-devkit/core", - "from": ">=21 <22", - "to": ">=22 <23", - "why": "4 breaking majors in two years (19, 20, 21, 22), 32,083,353 downloads a week", - "status": "todo", - "weeklyDownloads": 32083353, - "breakingMajors2y": 3 - }, { "package": "motion", "from": ">=13 <14", @@ -155,15 +137,6 @@ "weeklyDownloads": 28260335, "breakingMajors2y": 3 }, - { - "package": "@angular-devkit/schematics", - "from": ">=21 <22", - "to": ">=22 <23", - "why": "4 breaking majors in two years (19, 20, 21, 22), 27,747,217 downloads a week", - "status": "todo", - "weeklyDownloads": 27747217, - "breakingMajors2y": 3 - }, { "package": "stripe", "from": ">=22 <23", @@ -209,15 +182,6 @@ "weeklyDownloads": 63500483, "breakingMajors2y": 1 }, - { - "package": "jest-cli", - "from": ">=29 <30", - "to": ">=30 <31", - "why": "1 breaking major in two years (30), 56,018,663 downloads a week", - "status": "todo", - "weeklyDownloads": 56018663, - "breakingMajors2y": 1 - }, { "package": "emittery", "from": ">=1 <2", @@ -226,6 +190,42 @@ "status": "todo", "weeklyDownloads": 55768692, "breakingMajors2y": 1 + }, + { + "package": "react-router-dom", + "from": ">=6 <7", + "to": ">=7 <8", + "why": "1 breaking major in two years (7), 55,140,358 downloads a week", + "status": "todo", + "weeklyDownloads": 55140358, + "breakingMajors2y": 1 + }, + { + "package": "@expo/config-plugins", + "from": ">=56 <57", + "to": ">=57 <58", + "why": "7 breaking majors in two years (9, 10, 11, 54, 55, 56, 57), 18,154,913 downloads a week", + "status": "todo", + "weeklyDownloads": 18154913, + "breakingMajors2y": 3 + }, + { + "package": "@ai-sdk/openai", + "from": ">=3 <4", + "to": ">=4 <5", + "why": "4 breaking majors in two years (1, 2, 3, 4), 17,549,891 downloads a week", + "status": "todo", + "weeklyDownloads": 17549891, + "breakingMajors2y": 3 + }, + { + "package": "@inquirer/prompts", + "from": ">=7 <8", + "to": ">=8 <9", + "why": "1 breaking major in two years (8), 51,544,600 downloads a week", + "status": "todo", + "weeklyDownloads": 51544600, + "breakingMajors2y": 1 } ] } From 989d58cb485d082a6fc7d2067ced9a8f79b52bc7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 08:00:01 +0000 Subject: [PATCH 6/9] fix(packs): what got in the way writing the first pack with the tooling - pack new rendered the registry's imports in an order biome rejects when the new directory sorts before contract.ts; sorted by path now. - A rule can match on the compiler's message (a compiler-only finding's path is only its code), and the plan files a contract pack's sites under the rule that claims them, rewrite or not (ruleOf). - Fixture markers take message:"..."; pack test --update-fixtures writes after.ts from the rules; --fixtures-only says ground truth was skipped instead of "no repository yet". - Breaking recall counts a miss by its rule's severity. - A site found by detect carries the name the code writes. - Ground-truth repositories can install with bun (and the Packs job sets it up). - packs:truth links the target with its own dependencies at the versions it declares, and --also links what the upgrade moved with it; runs of changed lines split where the change does. - fix took its list of packs from two hard-coded lists (CLI and isolate); both ask the registry now. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01B2T8GmEVnZwfNrugajEMqi Signed-off-by: Claude --- .github/workflows/packs.yml | 3 + CONTRIBUTING.md | 5 +- docs/cli.md | 1 + docs/packs.md | 25 +++-- packages/cli/src/cli.ts | 27 ++++-- packages/cli/src/pack.ts | 3 +- packages/core/src/fetch/range.ts | 5 + packages/core/src/fix/isolate.ts | 3 +- packages/core/src/fix/plan.ts | 1 + packages/core/src/index.ts | 1 + packages/core/src/packs/contract.ts | 32 +++++- packages/core/src/packs/registry.ts | 1 + packages/core/src/packs/tooling/fixtures.ts | 30 ++++-- packages/core/src/packs/tooling/pack-test.ts | 13 ++- .../core/src/packs/tooling/registry-file.ts | 20 ++-- .../core/src/packs/tooling/scaffold.test.ts | 27 +++--- packages/core/src/packs/tooling/truth.ts | 22 ++++- packages/core/src/packs/types.ts | 6 ++ scripts/packs-truth.ts | 97 ++++++++++++++++--- 19 files changed, 259 insertions(+), 63 deletions(-) diff --git a/.github/workflows/packs.yml b/.github/workflows/packs.yml index 2d997c9..67a1ef2 100644 --- a/.github/workflows/packs.yml +++ b/.github/workflows/packs.yml @@ -44,6 +44,9 @@ jobs: cache: pnpm - if: steps.scope.outputs.run == 'true' run: pnpm install --frozen-lockfile --ignore-scripts + # Some ground-truth repositories install with bun (scripts off, as every install here). + - uses: oven-sh/setup-bun@v2 + if: steps.scope.outputs.run == 'true' # Ground-truth repositories (shallow, installed with scripts off) and the registry # tarballs `check` extracts. Keyed on the ground truth: a new repository is fetched once. - uses: actions/cache@v4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f4d7e8..8ba4bc4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -121,13 +121,16 @@ open a **Pack request** issue, and say you are writing it. `detect` when its sites can be found in the source. (15 minutes) 4. **Write fixtures**: `fixtures//before.ts` with each site marked (`// @uptide `, `keep` for one it must leave alone) and `after.ts`, what the rules - must produce. `pnpm uptide pack test --fixtures-only` runs them in seconds. + must produce. `pnpm uptide pack test --fixtures-only` runs them in seconds; + `--update-fixtures` writes each `after.ts` from what the rules do, for you to read. 5. **Find ground truth.** Two public repositories that made this upgrade in a commit you can point at. Pin the commit before it, and draft the expected findings from their own compiler and their own upgrade commit: ```sh pnpm packs:truth owner/name --to ``` + Add `--also @` for each package the upgrade commit moved together with + it (its lockfile has the versions). Read each site, drop what is not about the package, and give each its rule (`generic` when no rule of the pack covers it). (15 minutes) 6. **Score it.** diff --git a/docs/cli.md b/docs/cli.md index 65e4ab3..562608f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -327,6 +327,7 @@ pnpm uptide pack test --json # every pack, for CI pnpm uptide pack test ai --offline # cached repositories only pnpm uptide pack test ai --fixtures-only pnpm uptide pack test ai --write # record the measured status in verification.json +pnpm uptide pack test ai --fixtures-only --update-fixtures # write after.ts from the rules ``` `pack test` prints precision and recall per rule, overall and for breaking findings, and diff --git a/docs/packs.md b/docs/packs.md index 6f4ef24..ade004a 100644 --- a/docs/packs.md +++ b/docs/packs.md @@ -56,7 +56,9 @@ installed). - **Detect.** `check` already diffs the two versions' declarations and compiles the repository against the target. A rule claims the findings whose change kind and symbol - path it matches (`kinds`, `symbols`). For sites no type diff reports, a rule can add + path it matches (`kinds`, `symbols`), and, with `message`, what the compiler said there: a + compiler-only finding's path is just its code (`TS2353`), the message names the property. + The plan files every claimed site under the rule, whether or not it rewrites (`ruleOf`). For sites no type diff reports, a rule can add `detect(text, file)`, which returns sites found in the source of a file that uses the package; `check` lists them as the pack's own findings. - **Rewrite.** `rewrite(text, finding, context)` edits the reported site and nothing else, @@ -87,11 +89,14 @@ export const contact = z.string().email(); // @uptide string-format at:.email( p first non-blank column). - `kind:`, `path:`: the change `check` reports there, when the rule matches on them (default: the rule's first kind, and the `at:` text). +- `message:"..."`: the compiler's message there, for a rule that matches on `message`. Each marked site of a rule with `rewrite` is rewritten, bottom-up, and the result must equal `after.ts` byte for byte (the markers are comments, so they stay). A rule or note with `detect` must find exactly the marked lines in `before.ts`, and nothing on any other line. Fixtures are test data: biome and `tsc` skip them, and they are never reformatted. +`uptide pack test --fixtures-only --update-fixtures` writes each `after.ts` from +what the rules produce, like a snapshot update: read the diff before committing it. ### Behavior notes: what the compiler cannot see @@ -155,17 +160,24 @@ Where the expected findings come from: two sources that are not Uptide. target version linked where the package is installed and without it. Every diagnostic new at the target is a site: errors, and deprecations (6385, 6387, what an editor strikes through). A repository on TypeScript 7, which has no JavaScript API, is compiled with the - TypeScript of this checkout, and the draft says so. + TypeScript of this checkout, and the draft says so. The target is linked with its own + dependencies at the versions it declares, as an install would leave it, and + `--also @` links the companions the upgrade commit moved with it (the + versions in its lockfile): `ai` 7 next to `@ai-sdk/react` 3 is a mix no repository ships, + and its errors are not the upgrade's. 2. **The upgrade commit.** The lines it changed or removed (`git diff -U0 `, - the old side). One site per construct: a run of changed lines is one site, at its first - line of code, and a run that a diagnostic falls inside (or just after) is that - diagnostic's site. Comment-only lines, pure insertions (no line at ``), and + the old side). One site per change: consecutive changed lines of the same change are one + site, at the first line of code (a line that is plainly another rule's starts a new + one); a run with a diagnostic inside it, or right after a diagnostic of the same rule, is + that diagnostic's site. Comment-only lines, pure insertions (no line at ``), and changes for another package are not sites. `pnpm packs:truth` drafts both for one repository: ```sh pnpm packs:truth zod owner/name --to 4.6.5 [--directory web] +pnpm packs:truth ai vercel/chatbot --to 7.0.9 \ + --also @ai-sdk/react@4.0.10 --also @ai-sdk/provider@4.0.1 ``` It prints the compiler's new diagnostics with each line's text, and the changed and inserted @@ -181,7 +193,8 @@ version its lockfile has at `commit`, and `pack test` says so when it is not. They are scored like the others and never count toward `verified`. `uptide pack test` fetches each repository once: a shallow fetch of the pinned commit only, -dependencies installed with lifecycle scripts off, and nothing from the repository is ever +dependencies installed from its lockfile (npm, pnpm, Yarn or bun) with lifecycle scripts +off, and nothing from the repository is ever executed. They are kept in `~/.cache/uptide/ground-truth` (`UPTIDE_GROUND_TRUTH_CACHE`), so `pack test --offline` runs without network after the first time. `pnpm packs:fetch` fills the cache for every pack (and `--corpus` for the repositories in `fixtures/corpus.json`). diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 4f20aa8..a85d2d5 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs'; import { basename, resolve } from 'node:path'; import type { CheckReport, CheckResult, FixReport } from '@uptide/core'; import { + activePacks, formatFix, isFailure, PRICE_DATE, @@ -42,13 +43,7 @@ import { type Io, type Ui, type UiFlags, uiOf } from './io.js'; import { formatPackTest, packRoot, packsToTest } from './pack.js'; import { PRIVACY } from './privacy.js'; import { createProgress, elapsed, type Progress } from './progress.js'; -import { - collectStatus, - declaredDependencies, - formatStatus, - locateDependencies, - SUPPORTED as PACKED, -} from './status.js'; +import { collectStatus, declaredDependencies, formatStatus, locateDependencies } from './status.js'; import { createTelemetry, type Telemetry } from './telemetry/client.js'; import { checkMetrics, fixMetrics, listMetrics } from './telemetry/metrics.js'; import { VERSION } from './version.js'; @@ -712,7 +707,9 @@ ${EXIT_CODES('a plan was made', 'not used', ';\n or a dependency failed to a .description('Upgrade one dependency on a new branch and migrate your code, verified') .option( '--only ', - 'the dependency to upgrade: zod or stripe (verified), or any other (generic, agent only)', + `the dependency to upgrade: ${activePacks() + .map((pack) => pack.name) + .join(', ')} (verified), or any other (generic, agent only)`, ) .option( '--target ', @@ -830,7 +827,7 @@ ${PRIVACY}`, }); // No pack, so no rule: without the agent there is nothing this command can do, // and it says so before it touches anything. - const hasPack = (PACKED as readonly string[]).includes(only); + const hasPack = activePacks().some((pack) => pack.name === only); if (!hasPack && (flags.llm === false || !selection.available)) throw noAgentForGeneric(only, flags.llm === false); if (flags.pinCurrentApi && only !== 'stripe') @@ -1229,6 +1226,10 @@ ${EXIT_CODES('scaffolded', 'not used')}`, .option('--offline', 'never fetch: use only ground-truth repositories already cached') .option('--fixtures-only', 'run the fixtures, skip the ground-truth repositories') .option('--write', "record this run's result in the pack's verification.json") + .option( + '--update-fixtures', + "write each fixture's after.ts from what the rules produce; review the diff before committing", + ) .option('--cwd ', 'a directory inside the uptide checkout') .option('--json', 'machine-readable output for CI') .option('--no-color', 'no color (NO_COLOR is respected too)') @@ -1247,7 +1248,12 @@ ${EXIT_CODES('every pack passed', 'a false positive among breaking findings, a f .action( ( name: string | undefined, - flags: Shared & { offline?: boolean; fixturesOnly?: boolean; write?: boolean }, + flags: Shared & { + offline?: boolean; + fixturesOnly?: boolean; + write?: boolean; + updateFixtures?: boolean; + }, ) => act(flags, async ({ ui, cwd }) => { const root = packRoot(cwd); @@ -1258,6 +1264,7 @@ ${EXIT_CODES('every pack passed', 'a false positive among breaking findings, a f ...(flags.offline ? { offline: true } : {}), ...(flags.fixturesOnly ? { fixturesOnly: true } : {}), ...(flags.write ? { write: true } : {}), + ...(flags.updateFixtures ? { updateFixtures: true } : {}), log: (line) => io.err(`${line}\n`), }); reports.push(report); diff --git a/packages/cli/src/pack.ts b/packages/cli/src/pack.ts index 8f822fb..d143796 100644 --- a/packages/cli/src/pack.ts +++ b/packages/cli/src/pack.ts @@ -82,7 +82,8 @@ export function formatPackTest(report: PackTestReport, color = false): string { for (const s of f.falseNegatives) out.push(` ${c.red('✗')} ${s.file}:${s.line} ${s.rule} marked, not detected`); - if (report.repos.length === 0) + if (report.fixturesOnly) out.push('ground truth skipped (--fixtures-only)'); + else if (report.repos.length === 0) out.push('ground truth no repository yet: the pack ships as a candidate'); else { out.push('ground truth'); diff --git a/packages/core/src/fetch/range.ts b/packages/core/src/fetch/range.ts index bfe5cca..7b495ed 100644 --- a/packages/core/src/fetch/range.ts +++ b/packages/core/src/fetch/range.ts @@ -13,3 +13,8 @@ export function maxSatisfying(versions: readonly string[], range: string): strin export function minVersion(range: string): string | undefined { return semver.validRange(range) ? (semver.minVersion(range)?.version ?? undefined) : undefined; } + +/** Whether `range` is a semver range at all (`npm:`, `workspace:` and URLs are not). */ +export function validRange(range: string): boolean { + return semver.validRange(range) !== null; +} diff --git a/packages/core/src/fix/isolate.ts b/packages/core/src/fix/isolate.ts index ce923d9..7d81b75 100644 --- a/packages/core/src/fix/isolate.ts +++ b/packages/core/src/fix/isolate.ts @@ -17,6 +17,7 @@ import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; import { workspacePackagesOf } from '../adapters/typescript/repo.js'; import { UptideError } from '../errors.js'; import { ACCEPTED_KEYS, selectLlm } from '../llm/config.js'; +import { activePack } from '../packs/index.js'; import { uptideVersionInfo } from '../version.js'; import { git, projectRoot } from './process.js'; import { prBody } from './report.js'; @@ -456,7 +457,7 @@ export async function isolatedFix( throw new Error('--max-cost must be a positive finite amount in USD'); if ( !options.fixer && - !['zod', 'stripe'].includes(options.only) && + !activePack(options.only) && !options.pack?.rules.length && (options.fixer === null || !selected.available) ) diff --git a/packages/core/src/fix/plan.ts b/packages/core/src/fix/plan.ts index 83f2a6a..2bc1c8e 100644 --- a/packages/core/src/fix/plan.ts +++ b/packages/core/src/fix/plan.ts @@ -62,6 +62,7 @@ export function planPackage( const rule = finding.rule ?? dry?.rule ?? + (usable ? pack.ruleOf?.(finding) : undefined) ?? changeRule({ package: p.name }, { finding, outcome, reason: dry?.reason ?? finding.reason }); return { finding, outcome, rule }; }); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f24abf1..5676c4e 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -103,6 +103,7 @@ export { removePackageDir, } from './fetch/npm-fetcher.js'; export { loadRegistryConfig, type RegistryConfig } from './fetch/npmrc.js'; +export { maxSatisfying, satisfies, validRange } from './fetch/range.js'; export { cleanRuns, fixPreflight, diff --git a/packages/core/src/packs/contract.ts b/packages/core/src/packs/contract.ts index 6acb0b5..d4aa0b1 100644 --- a/packages/core/src/packs/contract.ts +++ b/packages/core/src/packs/contract.ts @@ -39,6 +39,8 @@ export interface SourceSite { column: number; /** The code at the site, trimmed: what the report shows. */ snippet: string; + /** The name as the code writes it (`system`, `stepCountIs`): what a summary of sites lists. */ + name?: string; } /** @@ -59,6 +61,12 @@ export interface PackRule extends MigrationRule { rewrite?(text: string, finding: Finding, context: PackContext): TransformResult; /** Sites in a file that uses the package which the type diff cannot report. */ detect?(text: string, file: string): SourceSite[]; + /** + * With `kinds` and `symbols`: what the compiler said at the site must match too. A + * compiler-only finding's path is its code (`TS2353`), which says nothing about which + * property was rejected; the message does. + */ + message?: RegExp; } /** @@ -184,14 +192,22 @@ export interface RegisteredPack { verification: PackVerification; } -/** The rule a finding belongs to: the one that set it, else the first whose kinds and symbols match. */ -export function ruleFor( +/** + * The rule a finding belongs to: the one that set it, else the first whose kinds, symbols and + * (when it has one) message match. + */ +export function ruleFor( rules: readonly R[], finding: Finding, ): R | undefined { return ( rules.find((r) => r.id === finding.rule) ?? - rules.find((r) => r.kinds.includes(finding.change.kind) && r.symbols.test(finding.change.path)) + rules.find( + (r) => + r.kinds.includes(finding.change.kind) && + r.symbols.test(finding.change.path) && + (!r.message || r.message.test(finding.usage.compileError ?? '')), + ) ); } @@ -223,7 +239,7 @@ function detected( const findings: Finding[] = []; for (const source of sources) for (const site of source.detect?.(input.text, input.file) ?? []) { - const path = `${pack.meta.package}:${source.id}`; + const path = site.name ?? `${pack.meta.package}:${source.id}`; findings.push({ change: { package: pack.meta.package, @@ -295,6 +311,7 @@ export function definePack(spec: PackSpec): Pack { const result = rule.rewrite(text, finding, context); return result.applied ? { ...result, rule: rule.id } : result; }, + ruleOf: (finding) => ruleFor([...spec.rules, ...detectingNotes(behavior)], finding)?.id, guide(finding) { const rule = ruleFor(spec.rules, finding); return [rule?.guide, spec.instructions].filter(Boolean).join('\n'); @@ -314,6 +331,13 @@ export function definePack(spec: PackSpec): Pack { return pack; } +/** Notes found by `detect`: their findings carry the note's id, as rules' do. */ +function detectingNotes(behavior: readonly BehaviorNote[]): MigrationRule[] { + return behavior + .filter((b) => b.detect && b.reported.includes('finding')) + .map((b) => ({ id: b.id, kinds: [], symbols: /$^/, guide: b.summary })); +} + /** Sites detected in one file, for fixtures and ground truth: what `check` would add. */ export function detectIn( pack: Pick, diff --git a/packages/core/src/packs/registry.ts b/packages/core/src/packs/registry.ts index 15a657f..b991804 100644 --- a/packages/core/src/packs/registry.ts +++ b/packages/core/src/packs/registry.ts @@ -1,5 +1,6 @@ // Generated by `uptide pack new` (packages/core/src/packs/tooling/registry-file.ts). // Every pack in this tree, with the status `uptide pack test --write` recorded for it. + import type { PackVerification, RegisteredPack } from './contract.js'; import { stripePack } from './stripe/index.js'; import stripeVerification from './stripe/verification.json' with { type: 'json' }; diff --git a/packages/core/src/packs/tooling/fixtures.ts b/packages/core/src/packs/tooling/fixtures.ts index 0cb3f2a..1fed56a 100644 --- a/packages/core/src/packs/tooling/fixtures.ts +++ b/packages/core/src/packs/tooling/fixtures.ts @@ -1,4 +1,4 @@ -import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import type { ChangeKind } from '../../domain/change.js'; import type { Finding } from '../../domain/report.js'; @@ -14,7 +14,8 @@ import { detectIn, type Pack, ruleFor } from '../contract.js'; * * `keep` marks a site the rule must leave alone. `at:` is where on the line the site * starts (default: the first non-blank column), `kind:` and `path:` the change `check` would - * report there (default: the rule's first kind, and the rule id). Every marked site of a rule + * report there (default: the rule's first kind, and the `at:` text), `message:"..."` the compiler's + * message for a compiler-only finding (a rule with `message` matches on it). Every marked site of a rule * with `rewrite` is rewritten, bottom-up, and the result must equal `after.ts` byte for byte * (markers included: they are comments). A rule or note with `detect` must find exactly its * marked lines in `before.ts`, and nothing on any other line. @@ -28,6 +29,7 @@ export interface FixtureSite { at?: string; kind?: ChangeKind; path?: string; + message?: string; } export interface FixtureResult { @@ -42,7 +44,7 @@ export interface FixtureResult { unknown: { file: string; line: number; rule: string }[]; } -const MARKER = /\/\/\s*@uptide\s+([\w-]+)((?:\s+[^\s]+)*)\s*$/; +const MARKER = /\/\/\s*@uptide\s+([\w-]+)((?:\s+(?:message:"[^"]*"|[^\s]+))*)\s*$/; export function parseMarkers(text: string, caseName: string, file: string): FixtureSite[] { const sites: FixtureSite[] = []; @@ -56,7 +58,11 @@ export function parseMarkers(text: string, caseName: string, file: string): Fixt rule: match[1] as string, keep: false, }; - for (const word of (match[2] ?? '').trim().split(/\s+/).filter(Boolean)) { + const rest = (match[2] ?? '').replace(/message:"([^"]*)"/, (_all, message: string) => { + site.message = message; + return ''; + }); + for (const word of rest.trim().split(/\s+/).filter(Boolean)) { if (word === 'keep') site.keep = true; else if (word.startsWith('at:')) site.at = word.slice(3); else if (word.startsWith('kind:')) site.kind = word.slice(5) as ChangeKind; @@ -109,6 +115,12 @@ export function fixtureFinding( access: 'call', snippet: lineText.trim(), via: 'direct', + ...(site.message !== undefined + ? { + compileError: site.message, + ...(/^TS\d+$/.test(path) ? { compileCode: Number(path.slice(2)) } : {}), + } + : {}), }, severity, confidence: 1, @@ -128,7 +140,12 @@ export function fixtureCases(dir: string): string[] { } /** Every fixture case of the pack in `dir`, scored the way ground truth is. */ -export function runFixtures(pack: Pack, dir: string): FixtureResult { +export function runFixtures( + pack: Pack, + dir: string, + /** Write each case's `after.ts` from what the rules produce (review the diff before committing). */ + options: { update?: boolean } = {}, +): FixtureResult { const result: FixtureResult = { cases: [], rules: {}, @@ -213,7 +230,8 @@ export function runFixtures(pack: Pack, dir: string): FixtureResult { if (!rule.detect) tally(rule.id).truePositives++; text = out.text; } - if (text !== after) + if (options.update && text !== after) writeFileSync(afterPath, text); + else if (text !== after) result.rewriteFailures.push({ case: name, file: `fixtures/${name}/after.ts`, diff --git a/packages/core/src/packs/tooling/pack-test.ts b/packages/core/src/packs/tooling/pack-test.ts index 6cbd748..c9d4f4a 100644 --- a/packages/core/src/packs/tooling/pack-test.ts +++ b/packages/core/src/packs/tooling/pack-test.ts @@ -63,6 +63,8 @@ export interface PackTestReport { recorded: PackStatus; /** The recorded verification no longer matches this run (`--write` updates it). */ stale: boolean; + /** Ground truth was not scored (`--fixtures-only`). */ + fixturesOnly: boolean; verification: PackVerification; problems: string[]; fixtures: FixtureResult; @@ -198,6 +200,8 @@ export interface PackTestOptions extends EnsureOptions { fixturesOnly?: boolean; /** Record this run's verification in the pack's `verification.json`. */ write?: boolean; + /** Write each fixture case's `after.ts` from what the rules produce. */ + updateFixtures?: boolean; } export function packsDir(root: string): string { @@ -221,7 +225,7 @@ export async function testPack( const problems: string[] = []; if (!pack.meta.sources.length) problems.push('meta.sources lists no changelog or guide'); if (!pack.meta.maintainer) problems.push('meta.maintainer is empty'); - const fixtures = runFixtures(pack, dir); + const fixtures = runFixtures(pack, dir, { update: options.updateFixtures === true }); if (fixtures.cases.length === 0 && pack.rules.some((r) => r.rewrite || r.detect)) problems.push('rules that rewrite or detect need fixtures (fixtures//before.ts)'); const read = readTruth(dir); @@ -242,10 +246,14 @@ export async function testPack( const overall = tally(predicted.length - fp.length, fp.length, fn.length); const breakingPredicted = predicted.filter((p) => p.severity === 'breaking'); const breakingFp = fp.filter((p) => p.severity === 'breaking'); + // A miss is a breaking one unless the rule it was expected under is deprecated. + const deprecatedRules = new Set( + pack.rules.filter((r) => r.severity === 'deprecated').map((r) => r.id), + ); const breaking = tally( breakingPredicted.length - breakingFp.length, breakingFp.length, - fn.length, + fn.filter((f) => !deprecatedRules.has(f.rule)).length, ); const rules: Record = {}; const ids = new Set([ @@ -297,6 +305,7 @@ export async function testPack( status, recorded, stale, + fixturesOnly: options.fixturesOnly === true, verification, problems, fixtures, diff --git a/packages/core/src/packs/tooling/registry-file.ts b/packages/core/src/packs/tooling/registry-file.ts index da7c332..ceb9ca7 100644 --- a/packages/core/src/packs/tooling/registry-file.ts +++ b/packages/core/src/packs/tooling/registry-file.ts @@ -15,18 +15,26 @@ const HEADER = `// Generated by \`uptide pack new\` (packages/core/src/packs/too export function renderRegistry(entries: readonly RegistryEntry[]): string { const sorted = [...entries].sort((a, b) => a.dir.localeCompare(b.dir)); - const imports = sorted.flatMap((e) => [ - `import { ${e.name} } from './${e.dir}/index.js';`, - `import ${verificationName(e)} from './${e.dir}/verification.json' with { type: 'json' };`, - ]); + // Sorted by module path as biome's organize-imports wants them, `./contract.js` included. + const imports: [string, string][] = [ + ['./contract.js', "import type { PackVerification, RegisteredPack } from './contract.js';"], + ...sorted.flatMap((e): [string, string][] => [ + [`./${e.dir}/index.js`, `import { ${e.name} } from './${e.dir}/index.js';`], + [ + `./${e.dir}/verification.json`, + `import ${verificationName(e)} from './${e.dir}/verification.json' with { type: 'json' };`, + ], + ]), + ]; + imports.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); const rows = sorted.map( (e) => ` { dir: '${e.dir}', pack: ${e.name}, verification: ${verificationName(e)} as PackVerification },`, ); return [ HEADER, - "import type { PackVerification, RegisteredPack } from './contract.js';", - ...imports, + '', + ...imports.map(([, line]) => line), '', 'export const REGISTRY: readonly RegisteredPack[] = [', ...rows, diff --git a/packages/core/src/packs/tooling/scaffold.test.ts b/packages/core/src/packs/tooling/scaffold.test.ts index 42dbf64..82d015c 100644 --- a/packages/core/src/packs/tooling/scaffold.test.ts +++ b/packages/core/src/packs/tooling/scaffold.test.ts @@ -45,16 +45,17 @@ function miniCheckout(): string { it('pack new scaffolds a pack that lints, passes its own test and pack test; one added rule keeps it passing', async () => { const home = miniCheckout(); - const result = scaffoldPack({ root, package: 'toy-lib', from: '>=1 <2', to: '>=2 <3' }); - expect(result).toMatchObject({ dir: 'toy-lib', constant: 'toyLibPack', formatted: true }); + const result = scaffoldPack({ root, package: 'alpha-lib', from: '>=1 <2', to: '>=2 <3' }); + expect(result).toMatchObject({ dir: 'alpha-lib', constant: 'alphaLibPack', formatted: true }); expect(parseRegistry(readFileSync(join(home, 'registry.ts'), 'utf8'))).toEqual([ - { dir: 'toy-lib', name: 'toyLibPack' }, + { dir: 'alpha-lib', name: 'alphaLibPack' }, ]); - expect(() => scaffoldPack({ root, package: 'toy-lib', from: '>=1 <2', to: '>=2 <3' })).toThrow( + expect(() => scaffoldPack({ root, package: 'alpha-lib', from: '>=1 <2', to: '>=2 <3' })).toThrow( /already exists/, ); - // Lint: the checkout's biome accepts every generated file as it is. + // Lint: the checkout's biome accepts every generated file as it is, the registry included + // (`alpha-lib` sorts before `contract.ts`, which the import order has to respect). const biome = spawnSync( join(repo, 'node_modules', '.bin', 'biome'), ['check', '--no-errors-on-unmatched', 'packages'], @@ -64,7 +65,7 @@ it('pack new scaffolds a pack that lints, passes its own test and pack test; one expect(biome.status).toBe(0); // Add one rule: a note the compiler cannot see, detected in the source, with its fixture. - const index = join(home, 'toy-lib', 'index.ts'); + const index = join(home, 'alpha-lib', 'index.ts'); writeFileSync( index, readFileSync(index, 'utf8').replace( @@ -85,11 +86,11 @@ it('pack new scaffolds a pack that lints, passes its own test and pack test; one ],`, ), ); - mkdirSync(join(home, 'toy-lib', 'fixtures', 'retry-default'), { recursive: true }); + mkdirSync(join(home, 'alpha-lib', 'fixtures', 'retry-default'), { recursive: true }); writeFileSync( - join(home, 'toy-lib', 'fixtures', 'retry-default', 'before.ts'), + join(home, 'alpha-lib', 'fixtures', 'retry-default', 'before.ts'), [ - "import { connect } from 'toy-lib';", + "import { connect } from 'alpha-lib';", '', "export const a = connect('db'); // @uptide retry-default", "export const b = connect('db', { retries: 0 }); // @uptide retry-default keep", @@ -97,11 +98,11 @@ it('pack new scaffolds a pack that lints, passes its own test and pack test; one ].join('\n'), ); - const pack = ((await import(index)) as { toyLibPack: Pack }).toyLibPack; + const pack = ((await import(index)) as { alphaLibPack: Pack }).alphaLibPack; const verification = JSON.parse( - readFileSync(join(home, 'toy-lib', 'verification.json'), 'utf8'), + readFileSync(join(home, 'alpha-lib', 'verification.json'), 'utf8'), ) as PackVerification; - const report = await testPack({ dir: 'toy-lib', pack, verification }, { root }); + const report = await testPack({ dir: 'alpha-lib', pack, verification }, { root }); expect(report.problems).toEqual([]); expect(report.fixtures.cases).toEqual(['example-rename', 'retry-default']); expect(report.fixtures.rules['retry-default']).toEqual({ @@ -115,7 +116,7 @@ it('pack new scaffolds a pack that lints, passes its own test and pack test; one // The generated test, run by the checkout's vitest as `pnpm test` would. const vitest = spawnSync( join(repo, 'node_modules', '.bin', 'vitest'), - ['run', '--root', join(root, 'packages', 'core'), 'src/packs/toy-lib'], + ['run', '--root', join(root, 'packages', 'core'), 'src/packs/alpha-lib'], { cwd: root, encoding: 'utf8' }, ); expect(vitest.stdout).toMatch(/1 passed/); diff --git a/packages/core/src/packs/tooling/truth.ts b/packages/core/src/packs/tooling/truth.ts index f1c1c09..4bb6d36 100644 --- a/packages/core/src/packs/tooling/truth.ts +++ b/packages/core/src/packs/tooling/truth.ts @@ -70,8 +70,28 @@ const UNFROZEN: Record = { pnpm: ['install', '--ignore-scripts', '--no-frozen-lockfile'], 'yarn-classic': ['install', '--ignore-scripts', '--non-interactive'], 'yarn-berry': ['install'], + bun: ['install', '--ignore-scripts'], }; +/** + * The package manager to install a ground-truth repository with: `fix`'s (npm, pnpm, Yarn), + * and bun, which `check` reads (its text lockfile) and some public repositories use. + */ +function installerOf(dir: string): { + kind: string; + bin: string; + args: string[]; + env?: Record; +} { + const others = ['pnpm-lock.yaml', 'package-lock.json', 'npm-shrinkwrap.json', 'yarn.lock']; + if ( + (existsSync(join(dir, 'bun.lock')) || existsSync(join(dir, 'bun.lockb'))) && + !others.some((file) => existsSync(join(dir, file))) + ) + return { kind: 'bun', bin: 'bun', args: ['install', '--frozen-lockfile', '--ignore-scripts'] }; + return packageManager(dir); +} + /** * The repository's own lockfile, frozen, with lifecycle scripts off. A repository whose * lockfile was already out of sync at that commit (it happens: `npm ci` refuses what `npm @@ -83,7 +103,7 @@ async function installDependencies( name: string, options: EnsureOptions, ): Promise<{ manager: string; frozen: boolean }> { - const pm = packageManager(dir); + const pm = installerOf(dir); const env = { ...pm.env, npm_config_manage_package_manager_versions: 'true' }; const timeout = options.timeoutMs ?? 900_000; options.log?.(`installing ${name} with ${pm.kind}, scripts off`); diff --git a/packages/core/src/packs/types.ts b/packages/core/src/packs/types.ts index 3b73920..404651d 100644 --- a/packages/core/src/packs/types.ts +++ b/packages/core/src/packs/types.ts @@ -99,6 +99,12 @@ export interface MigrationPack { supports(from: string, to: string): boolean; transform(text: string, finding: Finding, context: PackContext): TransformResult; guide(finding: Finding, context?: PackContext): string; + /** + * The rule a reported site belongs to, when the pack can say so itself: the plan files the + * site under it whether or not the rule rewrites. Packs built with `definePack` answer from + * their rules; without it the plan classifies by the change (`changeRule`). + */ + ruleOf?(finding: Finding): string | undefined; reviewNotes(context: PackContext): string[]; validateAssisted?( text: string, diff --git a/scripts/packs-truth.ts b/scripts/packs-truth.ts index aa828c9..5b1ff02 100644 --- a/scripts/packs-truth.ts +++ b/scripts/packs-truth.ts @@ -2,7 +2,8 @@ * A draft of the expected findings for one ground-truth repository, from two sources that are * not Uptide: the repository's own upgrade commit, and its own TypeScript compiler. * - * pnpm packs:truth --to [--directory ] + * pnpm packs:truth --to + * [--directory ] [--also @ ...] * * - changed: every line of `` the upgrade commit changed or removed (`git diff -U0`, * the old side), in source files. Pure insertions have no line in `` and are listed @@ -17,16 +18,21 @@ */ import { execFileSync } from 'node:child_process'; import { + cpSync, existsSync, lstatSync, + mkdirSync, + mkdtempSync, readdirSync, readFileSync, + realpathSync, renameSync, rmSync, symlinkSync, } from 'node:fs'; -import { join, relative } from 'node:path'; -import { createNpmFetcher, ensureRepo } from '@uptide/core'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join, relative } from 'node:path'; +import { createNpmFetcher, ensureRepo, maxSatisfying, satisfies, validRange } from '@uptide/core'; import { compilerDiagnostics, type Diagnostic } from './packs-truth-compiler.ts'; const args = process.argv.slice(2); @@ -87,14 +93,14 @@ for (const line of git('diff', '-U0', '--no-color', '--no-ext-diff', commit, upg } // Where the repository has the package: the root's node_modules and each workspace's. -function installs(dir: string, depth = 0): string[] { +function installs(name: string, dir: string, depth = 0): string[] { const found: string[] = []; - const own = join(dir, 'node_modules', ...pkg.split('/')); + const own = join(dir, 'node_modules', ...name.split('/')); if (existsSync(own)) found.push(relative(project, own)); if (depth >= 3) return found; for (const entry of readdirSync(dir, { withFileTypes: true })) if (entry.isDirectory() && !['node_modules', '.git', 'dist', 'build'].includes(entry.name)) - found.push(...installs(join(dir, entry.name), depth + 1)); + found.push(...installs(name, join(dir, entry.name), depth + 1)); return found; } function tsconfigs(dir: string, depth = 0): string[] { @@ -107,23 +113,90 @@ function tsconfigs(dir: string, depth = 0): string[] { found.push(...tsconfigs(join(dir, entry.name), depth + 1)); return found; } -const links = installs(project); const configs = tsconfigs(project); const fetcher = createNpmFetcher(); -const target = await fetcher.fetch(pkg, to); +const work = mkdtempSync(join(tmpdir(), 'uptide-truth-')); + +// The target, and the companions the upgrade moved with it (`--also @ai-sdk/react@4.0.10`). +const swaps: { name: string; version: string }[] = [ + { name: pkg, version: to }, + ...args + .filter((_a, i) => args[i - 1] === '--also') + .map((spec) => { + const at = spec.lastIndexOf('@'); + return { name: spec.slice(0, at), version: spec.slice(at + 1) }; + }), +]; +/** The version of `name` the repository resolves from `from` upward, with its real directory. */ +function consumerCopy(name: string, from: string): { version: string; dir: string } | undefined { + for (let dir = from; ; dir = dirname(dir)) { + const manifest = join(dir, 'node_modules', ...name.split('/'), 'package.json'); + if (existsSync(manifest)) + return { + version: JSON.parse(readFileSync(manifest, 'utf8')).version, + dir: realpathSync(dirname(manifest)), + }; + if (dirname(dir) === dir || !dir.startsWith(root)) return undefined; + } +} + +/** + * A package at a version, as an install of it would leave it: a copy with its own + * `node_modules`, each dependency the repository's copy when that satisfies the declared + * range (peers always the repository's), else the highest version inside the range, fetched + * and assembled the same way. What check does for the target (target-deps.ts). + */ +async function assemble(name: string, version: string, depth = 0): Promise { + const dir = join(work, `${name.replace('/', '__')}@${version}`); + if (existsSync(dir)) return dir; + cpSync((await fetcher.fetch(name, version)).dir, dir, { recursive: true }); + const manifest = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) as { + dependencies?: Record; + peerDependencies?: Record; + }; + const link = (dep: string, at: string) => { + const into = join(dir, 'node_modules', ...dep.split('/')); + mkdirSync(dirname(into), { recursive: true }); + if (!existsSync(into)) symlinkSync(at, into); + }; + for (const dep of Object.keys(manifest.peerDependencies ?? {})) { + const own = consumerCopy(dep, project); + if (own) link(dep, own.dir); + } + for (const [dep, range] of Object.entries(manifest.dependencies ?? {})) { + if (!validRange(range)) continue; + // A version the upgrade pinned (`--also`) wins, as one deduplicated copy in its lockfile does. + const pinned = swaps.find((swap) => swap.name === dep && satisfies(swap.version, range)); + const own = consumerCopy(dep, project); + if (pinned) link(dep, await assemble(pinned.name, pinned.version, depth + 1)); + else if (own && satisfies(own.version, range)) link(dep, own.dir); + else if (depth < 4) { + const best = maxSatisfying((await fetcher.versions?.(dep)) ?? [], range); + if (best) link(dep, await assemble(dep, best, depth + 1)); + } + } + return dir; +} + +const links: { at: string; dir: string }[] = []; +for (const swap of swaps) { + const dir = await assemble(swap.name, swap.version); + for (const at of installs(swap.name, project)) links.push({ at, dir }); +} const { compiler: compilerName, diagnostics: baseline } = compilerDiagnostics(project, configs); for (const link of links) { - const at = join(project, link); + const at = join(project, link.at); renameSync(at, `${at}.uptide-orig`); - symlinkSync(target.dir, at); + symlinkSync(link.dir, at); } let after: Diagnostic[]; try { after = compilerDiagnostics(project, configs).diagnostics; } finally { + rmSync(work, { recursive: true, force: true, maxRetries: 2 }); for (const link of links) { - const at = join(project, link); + const at = join(project, link.at); if (lstatSync(at).isSymbolicLink()) rmSync(at); renameSync(`${at}.uptide-orig`, at); } @@ -148,7 +221,7 @@ console.log( upgrade, ...(directory ? { directory } : {}), to, - links, + links: links.map((l) => `${l.at} → ${basename(l.dir)}`), compiledWith: compilerName, tsconfigs: configs, compiler: compiler.map((d) => ({ From 6367147447518b73eb80c110888aed594a520a01 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 08:00:02 +0000 Subject: [PATCH 7/9] test(packs): zod ground truth with the refined one-site-per-change rule Five more sites in PATH, all changes its upgrade made (a cast to a removed type, _def.typeName reads zod 4 no longer has). Still no false positive; recall recorded again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01B2T8GmEVnZwfNrugajEMqi Signed-off-by: Claude --- packages/core/src/packs/zod/ground-truth.json | 30 +++++++++++++++++++ packages/core/src/packs/zod/verification.json | 4 +-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/packages/core/src/packs/zod/ground-truth.json b/packages/core/src/packs/zod/ground-truth.json index 0e00e84..9729e37 100644 --- a/packages/core/src/packs/zod/ground-truth.json +++ b/packages/core/src/packs/zod/ground-truth.json @@ -313,6 +313,12 @@ "rule": "types", "note": "compiler TS6385" }, + { + "file": "packages/designer/src/open-workflow.ts", + "line": 149, + "rule": "generic", + "note": "changed by the upgrade" + }, { "file": "packages/engine/src/run-workflow.ts", "line": 1208, @@ -433,6 +439,12 @@ "rule": "generic", "note": "changed by the upgrade" }, + { + "file": "packages/schema/src/nodes.ts", + "line": 291, + "rule": "types", + "note": "changed by the upgrade" + }, { "file": "packages/schema/src/nodes.ts", "line": 292, @@ -475,6 +487,12 @@ "rule": "types", "note": "compiler TS6385" }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 86, + "rule": "generic", + "note": "changed by the upgrade" + }, { "file": "packages/schema/src/wire-step-plugins.ts", "line": 87, @@ -487,12 +505,24 @@ "rule": "types", "note": "compiler TS6385" }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 93, + "rule": "generic", + "note": "changed by the upgrade" + }, { "file": "packages/schema/src/wire-step-plugins.ts", "line": 94, "rule": "types", "note": "compiler TS6385" }, + { + "file": "packages/schema/src/wire-step-plugins.ts", + "line": 97, + "rule": "generic", + "note": "changed by the upgrade" + }, { "file": "packages/schema/src/wire-step-plugins.ts", "line": 104, diff --git a/packages/core/src/packs/zod/verification.json b/packages/core/src/packs/zod/verification.json index fb5186f..2ab89ad 100644 --- a/packages/core/src/packs/zod/verification.json +++ b/packages/core/src/packs/zod/verification.json @@ -1,10 +1,10 @@ { "status": "verified", - "truth": "sha256-ce97dc27050c267807a0765d9a472078fbcef84eb65877d5ecc1597be45f1957", + "truth": "sha256-076a6c920536242c855573c0875e1c6c9f9264e89f53ba46558c9db2d8a6a404", "repos": 3, "breaking": { "precision": 1, - "recall": 0.76, + "recall": 0.7125, "falsePositives": 0 } } From 4f836ccd827991b3cde590b0f97c83bf24f1c60d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 08:00:02 +0000 Subject: [PATCH 8/9] feat(packs): AI SDK (ai) 6 to 7, verified on vercel/chatbot and miurla/morphic Built with pack new and pack test only. Rules rewrite the renamed options (system, onFinish, onStepFinish, experimental_telemetry), stepCountIs, fullStream, totalUsage and the removed experimental_* options; telemetry metadata, tool context and the stream result helpers go to the agent; what the compiler cannot see is listed for review. Ground truth from each repository's own upgrade commit and compiler: no false positive, 47 of 61 sites (chatbot 27 of 27). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01B2T8GmEVnZwfNrugajEMqi Signed-off-by: Claude --- .changeset/ai-pack.md | 14 + README.md | 6 +- packages/cli/src/readme.test.ts | 2 +- packages/core/src/packs/ai/ai.test.ts | 13 + packages/core/src/packs/ai/detect.ts | 195 +++++++++ .../ai/fixtures/full-stream-clash/after.ts | 9 + .../ai/fixtures/full-stream-clash/before.ts | 9 + .../packs/ai/fixtures/full-stream/after.ts | 6 + .../packs/ai/fixtures/full-stream/before.ts | 6 + .../src/packs/ai/fixtures/options/after.ts | 37 ++ .../src/packs/ai/fixtures/options/before.ts | 37 ++ .../packs/ai/fixtures/removed-option/after.ts | 10 + .../ai/fixtures/removed-option/before.ts | 10 + .../src/packs/ai/fixtures/step-count/after.ts | 8 + .../packs/ai/fixtures/step-count/before.ts | 8 + .../packs/ai/fixtures/total-usage/after.ts | 8 + .../packs/ai/fixtures/total-usage/before.ts | 8 + packages/core/src/packs/ai/ground-truth.json | 392 ++++++++++++++++++ packages/core/src/packs/ai/index.ts | 278 +++++++++++++ packages/core/src/packs/ai/verification.json | 10 + packages/core/src/packs/registry.ts | 3 + 21 files changed, 1065 insertions(+), 4 deletions(-) create mode 100644 .changeset/ai-pack.md create mode 100644 packages/core/src/packs/ai/ai.test.ts create mode 100644 packages/core/src/packs/ai/detect.ts create mode 100644 packages/core/src/packs/ai/fixtures/full-stream-clash/after.ts create mode 100644 packages/core/src/packs/ai/fixtures/full-stream-clash/before.ts create mode 100644 packages/core/src/packs/ai/fixtures/full-stream/after.ts create mode 100644 packages/core/src/packs/ai/fixtures/full-stream/before.ts create mode 100644 packages/core/src/packs/ai/fixtures/options/after.ts create mode 100644 packages/core/src/packs/ai/fixtures/options/before.ts create mode 100644 packages/core/src/packs/ai/fixtures/removed-option/after.ts create mode 100644 packages/core/src/packs/ai/fixtures/removed-option/before.ts create mode 100644 packages/core/src/packs/ai/fixtures/step-count/after.ts create mode 100644 packages/core/src/packs/ai/fixtures/step-count/before.ts create mode 100644 packages/core/src/packs/ai/fixtures/total-usage/after.ts create mode 100644 packages/core/src/packs/ai/fixtures/total-usage/before.ts create mode 100644 packages/core/src/packs/ai/ground-truth.json create mode 100644 packages/core/src/packs/ai/index.ts create mode 100644 packages/core/src/packs/ai/verification.json diff --git a/.changeset/ai-pack.md b/.changeset/ai-pack.md new file mode 100644 index 0000000..705dff9 --- /dev/null +++ b/.changeset/ai-pack.md @@ -0,0 +1,14 @@ +--- +"uptide": minor +--- + +A verified migration pack for the AI SDK (`ai`) 6 → 7: the renamed options (`system` → +`instructions`, `onFinish` → `onEnd`, `onStepFinish` → `onStepEnd`, `experimental_telemetry` +→ `telemetry`), `stepCountIs` → `isStepCount`, `fullStream` → `stream`, `totalUsage` → `usage` +and the removed `experimental_*` options are rewritten by rule; telemetry `metadata`, tool +`context` and the stream result helpers go to the agent with the guide; what the compiler +cannot see (telemetry registration, results that now cover every step, rejected system +messages) is listed for review. Scored against vercel/chatbot and miurla/morphic at the +commit before their own upgrade: no false positive. `uptide pack test` gains +`--update-fixtures`, fixture markers take the compiler's `message`, and ground-truth +repositories can use bun. diff --git a/README.md b/README.md index bab53eb..ffe23cb 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ [![License: Apache 2.0](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) Uptide finds what a dependency upgrade breaks in your code, migrates it, and proves it with -your compiler and tests. TypeScript repositories; verified migrations for **zod 3 → 4** and -**stripe** today. +your compiler and tests. TypeScript repositories; verified migrations for **zod 3 → 4**, +**stripe** and the **AI SDK (`ai`) 6 → 7** today. ## Quickstart (30 seconds) @@ -205,7 +205,7 @@ Every direct dependency can be discovered and selected for analysis. What differ | Tier | Dependencies | What you get | | --- | --- | --- | -| **Verified** | zod 3 → 4, stripe 14 and newer | A migration pack: rules written for that dependency, a guide for the agent, behavior checks (zod schemas compared on generated inputs; Stripe changelog filtered to what you call), and ground truth the pack is scored against. `fix` migrates by rule first, by agent for the rest. | +| **Verified** | zod 3 → 4, stripe 14 and newer, ai 6 → 7 | A migration pack: rules written for that dependency, a guide for the agent, behavior checks (zod schemas compared on generated inputs; Stripe changelog filtered to what you call), and ground truth from public repositories the pack is scored against ([docs/packs.md](docs/packs.md)). `fix` migrates by rule first, by agent for the rest. | | **Generic** | any other dependency | The same analysis without a pack. A finding is called breaking only when your compiler or the runtime probe confirms it, or when it is a `require()` of an ESM-only package or an import of a removed export; everything else is in `--details`. `fix` migrates with the agent alone, under the same verification, and says so in the pull request. | The tier is on every row of `check`, in the HTML report and in the pull request. A generic diff --git a/packages/cli/src/readme.test.ts b/packages/cli/src/readme.test.ts index b9069c1..8032935 100644 --- a/packages/cli/src/readme.test.ts +++ b/packages/cli/src/readme.test.ts @@ -52,7 +52,7 @@ describe('README', () => { }); it('explains the support tiers, verification, privacy and exit codes', () => { - expect(readme).toMatch(/\| \*\*Verified\*\* \| zod 3 → 4, stripe 14 and newer \|/); + expect(readme).toMatch(/\| \*\*Verified\*\* \| zod 3 → 4, stripe 14 and newer, ai 6 → 7 \|/); expect(readme).toMatch(/\| \*\*Generic\*\* \| any other dependency/); for (const heading of ['## How verification works', '## Privacy', '## Documentation']) expect(readme).toContain(heading); diff --git a/packages/core/src/packs/ai/ai.test.ts b/packages/core/src/packs/ai/ai.test.ts new file mode 100644 index 0000000..d443ba5 --- /dev/null +++ b/packages/core/src/packs/ai/ai.test.ts @@ -0,0 +1,13 @@ +import { fileURLToPath } from 'node:url'; +import { expect, it } from 'vitest'; +import { runFixtures } from '../tooling/fixtures.js'; +import { aiPack } from './index.js'; + +it('ai: every fixture rewrites and detects exactly its marked sites', () => { + const result = runFixtures(aiPack, fileURLToPath(new URL('.', import.meta.url))); + expect(result.cases.length).toBeGreaterThan(0); + expect(result.unknown).toEqual([]); + expect(result.rewriteFailures).toEqual([]); + expect(result.falsePositives).toEqual([]); + expect(result.falseNegatives).toEqual([]); +}); diff --git a/packages/core/src/packs/ai/detect.ts b/packages/core/src/packs/ai/detect.ts new file mode 100644 index 0000000..03da772 --- /dev/null +++ b/packages/core/src/packs/ai/detect.ts @@ -0,0 +1,195 @@ +import { + type CallExpression, + type NewExpression, + Node, + type ObjectLiteralElementLike, + Project, + type SourceFile, + SyntaxKind, +} from 'ts-morph'; +import type { Finding } from '../../domain/report.js'; +import { onReset } from '../../shared-state.js'; +import type { SourceSite } from '../contract.js'; +import type { TransformResult } from '../types.js'; + +/** + * Where the AI SDK 7 renames apply: the options object of a call to a function imported from + * `ai`, by its exported name, and of the stream-result helpers, which no other library + * names this way. Text alone cannot tell `system:` in a `generateText` call from `system:` + * anywhere else; the import can. + */ +const parsed = new Map(); +onReset(() => parsed.clear()); +function parse(text: string): SourceFile { + let source = parsed.get(text); + if (!source) { + source = new Project({ useInMemoryFileSystem: true }).createSourceFile('consumer.tsx', text); + if (parsed.size >= 16) parsed.delete(parsed.keys().next().value as string); + parsed.set(text, source); + } + return source; +} + +/** Local names bound to `ai` exports: local → exported. */ +function imports(source: SourceFile): Map { + const names = new Map(); + for (const declaration of source.getImportDeclarations()) { + if (declaration.getModuleSpecifierValue() !== 'ai') continue; + for (const specifier of declaration.getNamedImports()) + names.set(specifier.getAliasNode()?.getText() ?? specifier.getName(), specifier.getName()); + } + return names; +} + +/** Methods of a streamText result that take UI message stream options. */ +const RESULT_METHODS = new Set([ + 'toUIMessageStream', + 'toUIMessageStreamResponse', + 'pipeUIMessageStreamToResponse', +]); + +/** The options object of each call to one of `callees` (exported names), with its callee. */ +function optionObjects( + source: SourceFile, + callees: ReadonlySet, +): { callee: string; options: ObjectLiteralElementLike[] }[] { + const names = imports(source); + const found: { callee: string; options: ObjectLiteralElementLike[] }[] = []; + const visit = (call: CallExpression | NewExpression) => { + const expression = call.getExpression(); + let callee: string | undefined; + if (Node.isIdentifier(expression)) callee = names.get(expression.getText()); + else if ( + Node.isPropertyAccessExpression(expression) && + RESULT_METHODS.has(expression.getName()) + ) + callee = expression.getName(); + if (!callee || !callees.has(callee)) return; + const argument = call.getArguments()[0]; + if (argument && Node.isObjectLiteralExpression(argument)) + found.push({ callee, options: argument.getProperties() }); + }; + for (const call of source.getDescendantsOfKind(SyntaxKind.CallExpression)) visit(call); + for (const call of source.getDescendantsOfKind(SyntaxKind.NewExpression)) visit(call); + return found; +} + +function keyOf(property: ObjectLiteralElementLike): Node | undefined { + if ( + Node.isPropertyAssignment(property) || + Node.isShorthandPropertyAssignment(property) || + Node.isMethodDeclaration(property) + ) { + const name = property.getNameNode(); + return Node.isIdentifier(name) || Node.isStringLiteral(name) ? name : undefined; + } + return undefined; +} + +function site(source: SourceFile, node: Node): SourceSite { + const { line, column } = source.getLineAndColumnAtPos(node.getStart()); + return { + line, + column, + snippet: (source.getFullText().split('\n')[line - 1] ?? '').trim(), + name: node.getText().replace(/['"]/g, ''), + }; +} + +/** Sites of the option `key` in calls to `callees`. */ +export function optionSites(text: string, key: string, callees: ReadonlySet): SourceSite[] { + if (!text.includes(key)) return []; + const source = parse(text); + return optionObjects(source, callees).flatMap(({ options }) => + options + .map(keyOf) + .filter((k): k is Node => k !== undefined && k.getText().replace(/['"]/g, '') === key) + .map((k) => site(source, k)), + ); +} + +/** The option key at the reported site renamed: `system:` → `instructions:`, `{ system }` → `{ instructions: system }`. */ +export function renameOption( + text: string, + finding: Pick, + key: string, + to: string, + callees: ReadonlySet, +): TransformResult { + const source = parse(text); + for (const { options } of optionObjects(source, callees)) + for (const property of options) { + const name = keyOf(property); + if (!name || name.getText().replace(/['"]/g, '') !== key) continue; + const at = site(source, name); + if (at.line !== finding.usage.line || at.column !== finding.usage.column) continue; + const replacement = Node.isShorthandPropertyAssignment(property) ? `${to}: ${key}` : to; + return { + text: `${text.slice(0, name.getStart())}${replacement}${text.slice(name.getEnd())}`, + applied: true, + reason: `\`${key}\` is \`${to}\` in AI SDK 7`, + }; + } + return { text, applied: false, reason: `the reported site is not the \`${key}\` option` }; +} + +/** `stepCountIs` imported from `ai`: the import and every reference to the local name. */ +export function importSites(text: string, exported: string): SourceSite[] { + if (!text.includes(exported)) return []; + const source = parse(text); + const sites: SourceSite[] = []; + for (const declaration of source.getImportDeclarations()) { + if (declaration.getModuleSpecifierValue() !== 'ai') continue; + for (const specifier of declaration.getNamedImports()) { + if (specifier.getName() !== exported) continue; + sites.push(site(source, specifier.getNameNode())); + // Aliased (`stepCountIs as stop`): the local keeps its name, only the import changes. + if (specifier.getAliasNode()) continue; + const local = specifier.getNameNode(); + if (!Node.isIdentifier(local)) continue; + for (const reference of local.findReferencesAsNodes()) + if ( + reference.getSourceFile() === source && + !reference.getFirstAncestorByKind(SyntaxKind.ImportDeclaration) + ) + sites.push(site(source, reference)); + } + } + return sites.sort((a, b) => a.line - b.line || a.column - b.column); +} + +/** + * Whether the file declares `name` anywhere (a variable, parameter, function, class or import), + * other than as a binding taken from a `streamText(...)` result: that one is what renaming + * `fullStream` to `stream` writes, so it never collides with the rename itself. + */ +export function declaresName(text: string, name: string): boolean { + if (!text.includes(name)) return false; + const source = parse(text); + const streamText = [...imports(source)].find(([, exported]) => exported === 'streamText')?.[0]; + return source.getDescendantsOfKind(SyntaxKind.Identifier).some((id) => { + if (id.getText() !== name) return false; + const parent = id.getParent(); + if (Node.isBindingElement(parent) && parent.getNameNode() === id) { + const declaration = parent.getFirstAncestorByKind(SyntaxKind.VariableDeclaration); + const init = declaration?.getInitializer(); + const call = init && Node.isAwaitExpression(init) ? init.getExpression() : init; + return !( + streamText && + call && + Node.isCallExpression(call) && + call.getExpression().getText() === streamText + ); + } + if (Node.isImportClause(parent)) return parent.getDefaultImport() === id; + return ( + (Node.isVariableDeclaration(parent) || + Node.isParameterDeclaration(parent) || + Node.isFunctionDeclaration(parent) || + Node.isClassDeclaration(parent) || + Node.isImportSpecifier(parent) || + Node.isNamespaceImport(parent)) && + parent.getNameNode() === id + ); + }); +} diff --git a/packages/core/src/packs/ai/fixtures/full-stream-clash/after.ts b/packages/core/src/packs/ai/fixtures/full-stream-clash/after.ts new file mode 100644 index 0000000..69c0c09 --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/full-stream-clash/after.ts @@ -0,0 +1,9 @@ +import { streamText } from 'ai'; + +// A `stream` already in the file: a destructured `fullStream` keeps its name. +export async function relay(stream: WritableStream, prompt: string) { + const { fullStream } = streamText({ model, prompt }); // @uptide full-stream keep at:fullStream kind:deprecated path:StreamTextResult#fullStream + const result = streamText({ model, prompt }); + for await (const part of result.stream) await write(stream, part); // @uptide full-stream at:.fullStream kind:deprecated path:StreamTextResult#fullStream + return fullStream; +} diff --git a/packages/core/src/packs/ai/fixtures/full-stream-clash/before.ts b/packages/core/src/packs/ai/fixtures/full-stream-clash/before.ts new file mode 100644 index 0000000..09eb482 --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/full-stream-clash/before.ts @@ -0,0 +1,9 @@ +import { streamText } from 'ai'; + +// A `stream` already in the file: a destructured `fullStream` keeps its name. +export async function relay(stream: WritableStream, prompt: string) { + const { fullStream } = streamText({ model, prompt }); // @uptide full-stream keep at:fullStream kind:deprecated path:StreamTextResult#fullStream + const result = streamText({ model, prompt }); + for await (const part of result.fullStream) await write(stream, part); // @uptide full-stream at:.fullStream kind:deprecated path:StreamTextResult#fullStream + return fullStream; +} diff --git a/packages/core/src/packs/ai/fixtures/full-stream/after.ts b/packages/core/src/packs/ai/fixtures/full-stream/after.ts new file mode 100644 index 0000000..d9fc11d --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/full-stream/after.ts @@ -0,0 +1,6 @@ +import { streamText } from 'ai'; + +export async function* parts(prompt: string) { + const { stream } = streamText({ model, prompt }); // @uptide full-stream at:fullStream kind:deprecated path:StreamTextResult#fullStream + for await (const part of stream) yield part; // @uptide full-stream at:fullStream kind:deprecated path:StreamTextResult#fullStream +} diff --git a/packages/core/src/packs/ai/fixtures/full-stream/before.ts b/packages/core/src/packs/ai/fixtures/full-stream/before.ts new file mode 100644 index 0000000..93d3989 --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/full-stream/before.ts @@ -0,0 +1,6 @@ +import { streamText } from 'ai'; + +export async function* parts(prompt: string) { + const { fullStream } = streamText({ model, prompt }); // @uptide full-stream at:fullStream kind:deprecated path:StreamTextResult#fullStream + for await (const part of fullStream) yield part; // @uptide full-stream at:fullStream kind:deprecated path:StreamTextResult#fullStream +} diff --git a/packages/core/src/packs/ai/fixtures/options/after.ts b/packages/core/src/packs/ai/fixtures/options/after.ts new file mode 100644 index 0000000..7d96286 --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/options/after.ts @@ -0,0 +1,37 @@ +import { useChat } from '@ai-sdk/react'; +import { createUIMessageStream, generateObject, generateText as generate, streamText, ToolLoopAgent } from 'ai'; +import { complete } from 'other-llm'; + +const system = 'You are terse.'; + +export async function title(prompt: string) { + const { text } = await generate({ + model, + instructions: 'Write a title.', // @uptide instructions + prompt, + onStepEnd: (step) => log(step), // @uptide on-step-end + telemetry: { isEnabled: true, functionId: 'title' }, // @uptide telemetry + onEnd: ({ text }) => log(text), // @uptide on-end + }); + return text; +} + +export const reply = streamText({ model, instructions: system, prompt: 'hi' }); // @uptide instructions at:system + +export const agent = new ToolLoopAgent({ + model, + instructions: 'Use the tools.', // @uptide instructions + onEnd() {}, // @uptide on-end +}); + +export const ui = createUIMessageStream({ + execute: ({ writer }) => writer.merge(reply.toUIMessageStream({ onEnd: () => {} })), // @uptide on-end at:onFinish + onEnd: async ({ messages }) => save(messages), // @uptide on-end +}); + +// Left alone: another library's options, a hook from @ai-sdk/react, generateObject's onFinish. +export const other = complete({ system: 'x' }); // @uptide instructions keep at:system +export const chat = useChat({ onFinish: () => {} }); // @uptide on-end keep at:onFinish +export const object = generateObject({ model, schema, prompt: 'p', onFinish: () => {} }); // @uptide on-end keep at:onFinish +export const traced = complete({ onStepFinish: log, experimental_telemetry: on }); // @uptide on-step-end keep at:onStepFinish +export const tracedToo = complete({ experimental_telemetry: on }); // @uptide telemetry keep at:experimental_telemetry diff --git a/packages/core/src/packs/ai/fixtures/options/before.ts b/packages/core/src/packs/ai/fixtures/options/before.ts new file mode 100644 index 0000000..00a7b2b --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/options/before.ts @@ -0,0 +1,37 @@ +import { useChat } from '@ai-sdk/react'; +import { createUIMessageStream, generateObject, generateText as generate, streamText, ToolLoopAgent } from 'ai'; +import { complete } from 'other-llm'; + +const system = 'You are terse.'; + +export async function title(prompt: string) { + const { text } = await generate({ + model, + system: 'Write a title.', // @uptide instructions + prompt, + onStepFinish: (step) => log(step), // @uptide on-step-end + experimental_telemetry: { isEnabled: true, functionId: 'title' }, // @uptide telemetry + onFinish: ({ text }) => log(text), // @uptide on-end + }); + return text; +} + +export const reply = streamText({ model, system, prompt: 'hi' }); // @uptide instructions at:system + +export const agent = new ToolLoopAgent({ + model, + system: 'Use the tools.', // @uptide instructions + onFinish() {}, // @uptide on-end +}); + +export const ui = createUIMessageStream({ + execute: ({ writer }) => writer.merge(reply.toUIMessageStream({ onFinish: () => {} })), // @uptide on-end at:onFinish + onFinish: async ({ messages }) => save(messages), // @uptide on-end +}); + +// Left alone: another library's options, a hook from @ai-sdk/react, generateObject's onFinish. +export const other = complete({ system: 'x' }); // @uptide instructions keep at:system +export const chat = useChat({ onFinish: () => {} }); // @uptide on-end keep at:onFinish +export const object = generateObject({ model, schema, prompt: 'p', onFinish: () => {} }); // @uptide on-end keep at:onFinish +export const traced = complete({ onStepFinish: log, experimental_telemetry: on }); // @uptide on-step-end keep at:onStepFinish +export const tracedToo = complete({ experimental_telemetry: on }); // @uptide telemetry keep at:experimental_telemetry diff --git a/packages/core/src/packs/ai/fixtures/removed-option/after.ts b/packages/core/src/packs/ai/fixtures/removed-option/after.ts new file mode 100644 index 0000000..4162f46 --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/removed-option/after.ts @@ -0,0 +1,10 @@ +import { smoothStream, streamText } from 'ai'; + +export const result = streamText({ + model, + prompt: 'p', + activeTools: ['search'], // @uptide removed-experimental-option at:experimental_activeTools kind:type path:TS2353 message:"Object literal may only specify known properties, and 'experimental_activeTools' does not exist in type" + prepareStep: prepare, // @uptide removed-experimental-option at:experimental_prepareStep kind:type path:TS2353 message:"Object literal may only specify known properties, and 'experimental_prepareStep' does not exist in type" + // Another rejected option is another rule's: never renamed by this one. + experimental_transform: smoothStream(), // @uptide removed-experimental-option keep at:experimental_transform kind:type path:TS2353 message:"Object literal may only specify known properties, and 'metadata' does not exist in type" +}); diff --git a/packages/core/src/packs/ai/fixtures/removed-option/before.ts b/packages/core/src/packs/ai/fixtures/removed-option/before.ts new file mode 100644 index 0000000..796f2c7 --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/removed-option/before.ts @@ -0,0 +1,10 @@ +import { smoothStream, streamText } from 'ai'; + +export const result = streamText({ + model, + prompt: 'p', + experimental_activeTools: ['search'], // @uptide removed-experimental-option at:experimental_activeTools kind:type path:TS2353 message:"Object literal may only specify known properties, and 'experimental_activeTools' does not exist in type" + experimental_prepareStep: prepare, // @uptide removed-experimental-option at:experimental_prepareStep kind:type path:TS2353 message:"Object literal may only specify known properties, and 'experimental_prepareStep' does not exist in type" + // Another rejected option is another rule's: never renamed by this one. + experimental_transform: smoothStream(), // @uptide removed-experimental-option keep at:experimental_transform kind:type path:TS2353 message:"Object literal may only specify known properties, and 'metadata' does not exist in type" +}); diff --git a/packages/core/src/packs/ai/fixtures/step-count/after.ts b/packages/core/src/packs/ai/fixtures/step-count/after.ts new file mode 100644 index 0000000..85ea070 --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/step-count/after.ts @@ -0,0 +1,8 @@ +import { isStepCount, streamText } from 'ai'; // @uptide is-step-count at:stepCountIs +import { isStepCount as stopAfter } from 'ai'; // @uptide is-step-count at:stepCountIs + +export const a = streamText({ model, prompt: 'p', stopWhen: isStepCount(5) }); // @uptide is-step-count at:stepCountIs +export const b = streamText({ model, prompt: 'p', stopWhen: [isStepCount(10), done] }); // @uptide is-step-count at:stepCountIs +// The alias keeps its name; a name that only starts the same is not the import. +export const c = streamText({ model, prompt: 'p', stopWhen: stopAfter(3) }); +export const stepCountIsDefault = 5; // @uptide is-step-count keep at:stepCountIsDefault diff --git a/packages/core/src/packs/ai/fixtures/step-count/before.ts b/packages/core/src/packs/ai/fixtures/step-count/before.ts new file mode 100644 index 0000000..9e08892 --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/step-count/before.ts @@ -0,0 +1,8 @@ +import { stepCountIs, streamText } from 'ai'; // @uptide is-step-count at:stepCountIs +import { stepCountIs as stopAfter } from 'ai'; // @uptide is-step-count at:stepCountIs + +export const a = streamText({ model, prompt: 'p', stopWhen: stepCountIs(5) }); // @uptide is-step-count at:stepCountIs +export const b = streamText({ model, prompt: 'p', stopWhen: [stepCountIs(10), done] }); // @uptide is-step-count at:stepCountIs +// The alias keeps its name; a name that only starts the same is not the import. +export const c = streamText({ model, prompt: 'p', stopWhen: stopAfter(3) }); +export const stepCountIsDefault = 5; // @uptide is-step-count keep at:stepCountIsDefault diff --git a/packages/core/src/packs/ai/fixtures/total-usage/after.ts b/packages/core/src/packs/ai/fixtures/total-usage/after.ts new file mode 100644 index 0000000..7ec07cb --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/total-usage/after.ts @@ -0,0 +1,8 @@ +import { streamText } from 'ai'; + +export async function cost(prompt: string) { + const result = streamText({ model, prompt }); + const used = await result.usage; // @uptide total-usage at:.totalUsage kind:deprecated path:StreamTextResult#totalUsage + const totalUsage = used.inputTokens; // @uptide total-usage keep at:totalUsage kind:deprecated path:StreamTextResult#totalUsage + return totalUsage; +} diff --git a/packages/core/src/packs/ai/fixtures/total-usage/before.ts b/packages/core/src/packs/ai/fixtures/total-usage/before.ts new file mode 100644 index 0000000..d44ef48 --- /dev/null +++ b/packages/core/src/packs/ai/fixtures/total-usage/before.ts @@ -0,0 +1,8 @@ +import { streamText } from 'ai'; + +export async function cost(prompt: string) { + const result = streamText({ model, prompt }); + const used = await result.totalUsage; // @uptide total-usage at:.totalUsage kind:deprecated path:StreamTextResult#totalUsage + const totalUsage = used.inputTokens; // @uptide total-usage keep at:totalUsage kind:deprecated path:StreamTextResult#totalUsage + return totalUsage; +} diff --git a/packages/core/src/packs/ai/ground-truth.json b/packages/core/src/packs/ai/ground-truth.json new file mode 100644 index 0000000..c08d252 --- /dev/null +++ b/packages/core/src/packs/ai/ground-truth.json @@ -0,0 +1,392 @@ +{ + "$comment": "Ground truth for the ai pack (docs/packs.md, \"Ground truth\"). Each public repository is pinned at the commit before its own AI SDK 6 to 7 upgrade. Expected findings come from two sources that are not Uptide, drafted by pnpm packs:truth and read by hand: the diagnostics the repository's TypeScript reports with the target and the companions its upgrade moved with it linked in (@ai-sdk/react, @ai-sdk/provider, the providers; their versions from the upgrade's lockfile) and not without them, errors and deprecations, and the lines the upgrade commit changed for ai (one site per construct; comments, insertions and other packages' changes such as @ai-sdk/react's experimental_throttle left out). Rules are assigned from each site's code and text.", + "package": "ai", + "repos": [ + { + "repo": "vercel/chatbot", + "commit": "77d362e596e2c624638cad5a35199c0c8785c9fd", + "migration": "https://github.com/vercel/chatbot/commit/2397c783b505997a09968c4619ec42dc450cc705", + "from": "6.0.116", + "to": "7.0.9", + "why": "Next.js chat app: streamText with tools, artifact streams over fullStream, UI message streams, telemetry", + "findings": [ + { + "file": "app/(chat)/actions.ts", + "line": 30, + "rule": "instructions", + "note": "changed by the upgrade" + }, + { + "file": "app/(chat)/api/chat/route.ts", + "line": 7, + "rule": "is-step-count", + "note": "changed by the upgrade" + }, + { + "file": "app/(chat)/api/chat/route.ts", + "line": 196, + "rule": "instructions", + "note": "changed by the upgrade" + }, + { + "file": "app/(chat)/api/chat/route.ts", + "line": 198, + "rule": "is-step-count", + "note": "changed by the upgrade" + }, + { + "file": "app/(chat)/api/chat/route.ts", + "line": 199, + "rule": "removed-experimental-option", + "note": "compiler TS2353" + }, + { + "file": "app/(chat)/api/chat/route.ts", + "line": 236, + "rule": "telemetry", + "note": "changed by the upgrade" + }, + { + "file": "app/(chat)/api/chat/route.ts", + "line": 243, + "rule": "ui-message-stream", + "note": "compiler TS6387" + }, + { + "file": "app/(chat)/api/chat/route.ts", + "line": 257, + "rule": "on-end", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/code/server.ts", + "line": 18, + "rule": "full-stream", + "note": "compiler TS6385" + }, + { + "file": "artifacts/code/server.ts", + "line": 20, + "rule": "instructions", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/code/server.ts", + "line": 24, + "rule": "full-stream", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/code/server.ts", + "line": 40, + "rule": "full-stream", + "note": "compiler TS6385" + }, + { + "file": "artifacts/code/server.ts", + "line": 42, + "rule": "instructions", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/code/server.ts", + "line": 46, + "rule": "full-stream", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/sheet/server.ts", + "line": 11, + "rule": "full-stream", + "note": "compiler TS6385" + }, + { + "file": "artifacts/sheet/server.ts", + "line": 13, + "rule": "instructions", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/sheet/server.ts", + "line": 17, + "rule": "full-stream", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/sheet/server.ts", + "line": 33, + "rule": "full-stream", + "note": "compiler TS6385" + }, + { + "file": "artifacts/sheet/server.ts", + "line": 35, + "rule": "instructions", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/sheet/server.ts", + "line": 39, + "rule": "full-stream", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/text/server.ts", + "line": 11, + "rule": "full-stream", + "note": "compiler TS6385" + }, + { + "file": "artifacts/text/server.ts", + "line": 13, + "rule": "instructions", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/text/server.ts", + "line": 19, + "rule": "full-stream", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/text/server.ts", + "line": 35, + "rule": "full-stream", + "note": "compiler TS6385" + }, + { + "file": "artifacts/text/server.ts", + "line": 37, + "rule": "instructions", + "note": "changed by the upgrade" + }, + { + "file": "artifacts/text/server.ts", + "line": 42, + "rule": "full-stream", + "note": "changed by the upgrade" + }, + { + "file": "lib/ai/tools/request-suggestions.ts", + "line": 51, + "rule": "instructions", + "note": "changed by the upgrade" + } + ] + }, + { + "repo": "miurla/morphic", + "commit": "989a05697d973030f2fbb4b805326bb24e68fd0d", + "migration": "https://github.com/miurla/morphic/commit/0ec2ca733c8cadb589d741097a2766f67e1766a3", + "from": "6.0.180", + "to": "7.0.66", + "why": "Next.js research agent on bun: ToolLoopAgent, telemetry metadata moved to runtimeContext, tool execute options, UI message streams, tests that mock ai", + "findings": [ + { + "file": "lib/agents/__tests__/researcher.test.ts", + "line": 10, + "rule": "is-step-count", + "note": "changed by the upgrade" + }, + { + "file": "lib/agents/researcher.ts", + "line": 1, + "rule": "is-step-count", + "note": "changed by the upgrade" + }, + { + "file": "lib/agents/researcher.ts", + "line": 3, + "rule": "generic", + "note": "changed by the upgrade" + }, + { + "file": "lib/agents/researcher.ts", + "line": 145, + "rule": "is-step-count", + "note": "changed by the upgrade" + }, + { + "file": "lib/agents/researcher.ts", + "line": 148, + "rule": "telemetry", + "note": "changed by the upgrade" + }, + { + "file": "lib/agents/researcher.ts", + "line": 151, + "rule": "telemetry-metadata", + "note": "compiler TS2353" + }, + { + "file": "lib/agents/researcher.ts", + "line": 168, + "rule": "generic", + "note": "changed by the upgrade" + }, + { + "file": "lib/agents/title-generator.ts", + "line": 43, + "rule": "instructions", + "note": "changed by the upgrade" + }, + { + "file": "lib/agents/title-generator.ts", + "line": 48, + "rule": "telemetry", + "note": "changed by the upgrade" + }, + { + "file": "lib/agents/title-generator.ts", + "line": 51, + "rule": "telemetry-metadata", + "note": "compiler TS2353" + }, + { + "file": "lib/render/__tests__/llm-image-output.e2e.test.ts", + "line": 12, + "rule": "is-step-count", + "note": "changed by the upgrade" + }, + { + "file": "lib/render/__tests__/llm-image-output.e2e.test.ts", + "line": 89, + "rule": "instructions", + "note": "changed by the upgrade" + }, + { + "file": "lib/render/__tests__/llm-image-output.e2e.test.ts", + "line": 93, + "rule": "is-step-count", + "note": "changed by the upgrade" + }, + { + "file": "lib/render/__tests__/llm-related-questions.e2e.test.ts", + "line": 11, + "rule": "is-step-count", + "note": "changed by the upgrade" + }, + { + "file": "lib/render/__tests__/llm-related-questions.e2e.test.ts", + "line": 103, + "rule": "instructions", + "note": "changed by the upgrade" + }, + { + "file": "lib/render/__tests__/llm-related-questions.e2e.test.ts", + "line": 107, + "rule": "is-step-count", + "note": "changed by the upgrade" + }, + { + "file": "lib/streaming/__tests__/create-chat-stream-response.test.ts", + "line": 79, + "rule": "on-end", + "note": "changed by the upgrade" + }, + { + "file": "lib/streaming/__tests__/create-chat-stream-response.test.ts", + "line": 94, + "rule": "on-end", + "note": "changed by the upgrade" + }, + { + "file": "lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts", + "line": 62, + "rule": "on-end", + "note": "changed by the upgrade" + }, + { + "file": "lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts", + "line": 88, + "rule": "on-end", + "note": "changed by the upgrade" + }, + { + "file": "lib/streaming/create-chat-stream-response.ts", + "line": 222, + "rule": "on-step-end", + "note": "changed by the upgrade" + }, + { + "file": "lib/streaming/create-chat-stream-response.ts", + "line": 238, + "rule": "total-usage", + "note": "compiler TS6385" + }, + { + "file": "lib/streaming/create-chat-stream-response.ts", + "line": 245, + "rule": "ui-message-stream", + "note": "compiler TS6387" + }, + { + "file": "lib/streaming/create-chat-stream-response.ts", + "line": 255, + "rule": "on-end", + "note": "changed by the upgrade" + }, + { + "file": "lib/streaming/create-ephemeral-chat-stream-response.ts", + "line": 134, + "rule": "on-step-end", + "note": "changed by the upgrade" + }, + { + "file": "lib/streaming/create-ephemeral-chat-stream-response.ts", + "line": 148, + "rule": "total-usage", + "note": "compiler TS6385" + }, + { + "file": "lib/streaming/create-ephemeral-chat-stream-response.ts", + "line": 153, + "rule": "ui-message-stream", + "note": "compiler TS6387" + }, + { + "file": "lib/streaming/create-ephemeral-chat-stream-response.ts", + "line": 163, + "rule": "on-end", + "note": "changed by the upgrade" + }, + { + "file": "lib/tools/__tests__/fetch-error.test.ts", + "line": 23, + "rule": "tool-context", + "note": "compiler TS2345" + }, + { + "file": "lib/tools/__tests__/fetch-error.test.ts", + "line": 55, + "rule": "tool-context", + "note": "compiler TS2345" + }, + { + "file": "lib/tools/__tests__/fetch-error.test.ts", + "line": 91, + "rule": "tool-context", + "note": "compiler TS2345" + }, + { + "file": "lib/tools/search.ts", + "line": 212, + "rule": "tool-context", + "note": "compiler TS2345" + }, + { + "file": "lib/tools/search.ts", + "line": 214, + "rule": "generic", + "note": "changed by the upgrade" + }, + { + "file": "lib/types/agent.ts", + "line": 23, + "rule": "generic", + "note": "changed by the upgrade" + } + ] + } + ] +} diff --git a/packages/core/src/packs/ai/index.ts b/packages/core/src/packs/ai/index.ts new file mode 100644 index 0000000..f4cb30b --- /dev/null +++ b/packages/core/src/packs/ai/index.ts @@ -0,0 +1,278 @@ +import { definePack, type PackRule, replaceAtSite } from '../contract.js'; +import type { TransformResult } from '../types.js'; +import { declaresName, importSites, optionSites, renameOption } from './detect.js'; + +/** + * AI SDK (`ai`) 6 → 7. Taken from the official migration guide and the 7.0.0 changelog + * (`meta.sources`), checked 2026-10-07 against ai 7.0.130. Every rename below is a deprecated + * alias that still compiles in 7.x, so its rule is `deprecated`; what the compiler rejects is + * `breaking`. Changes the compiler cannot see are behavior notes, listed for review. + */ +const CALLS = new Set([ + 'generateText', + 'streamText', + 'generateObject', + 'streamObject', + 'ToolLoopAgent', +]); +const TEXT_CALLS = new Set(['generateText', 'streamText', 'ToolLoopAgent']); +const UI_STREAMS = new Set([ + 'createUIMessageStream', + 'toUIMessageStream', + 'toUIMessageStreamResponse', + 'pipeUIMessageStreamToResponse', +]); +const TELEMETRY_CALLS = new Set([...CALLS, 'embed', 'embedMany', 'rerank']); + +/** A renamed option of the calls above: found by `detect`, renamed by `rewrite`. */ +function option( + id: string, + key: string, + to: string, + callees: ReadonlySet, + summary: string, +): PackRule { + return { + id, + summary, + severity: 'deprecated', + // Sites come from `detect`: no type diff reports an option key that still compiles. + kinds: [], + symbols: /$^/, + guide: `${summary}. Rename the option and keep its value as it is.`, + detect: (text) => optionSites(text, key, callees), + rewrite: (text, finding) => renameOption(text, finding, key, to, callees), + }; +} + +/** Where a line's trailing `//` comment starts, outside strings; the line's length without one. */ +function codeEnd(line: string): number { + let quote: string | undefined; + for (let i = 0; i < line.length; i++) { + const c = line[i]; + if (quote) { + if (c === '\\') i++; + else if (c === quote) quote = undefined; + } else if (c === '"' || c === "'" || c === '`') quote = c; + else if (c === '/' && line[i + 1] === '/') return i; + } + return line.length; +} + +/** One occurrence of `before` in the code of the reported line, when it has exactly one. */ +function onlyOnLine( + text: string, + line: number, + before: RegExp, + after: string, + reason: string, +): TransformResult { + const lines = text.split('\n'); + const current = lines[line - 1] ?? ''; + const end = codeEnd(current); + const code = current.slice(0, end); + const matches = code.match(new RegExp(before.source, 'g')) ?? []; + if (matches.length !== 1) + return { text, applied: false, reason: 'the reported line is not one occurrence to rename' }; + lines[line - 1] = `${code.replace(before, after)}${current.slice(end)}`; + return { text: lines.join('\n'), applied: true, reason }; +} + +export const aiPack = definePack({ + meta: { + package: 'ai', + from: '>=6 <7', + to: '>=7 <8', + sources: [ + { + title: 'Migrate AI SDK 6.x to 7.0', + url: 'https://ai-sdk.dev/docs/migration-guides/migration-guide-7-0', + }, + { + title: 'Migration guide source', + url: 'https://github.com/vercel/ai/blob/main/content/docs/08-migration-guides/23-migration-guide-7-0.mdx', + }, + { + title: 'ai 7.0.0 changelog', + url: 'https://github.com/vercel/ai/blob/main/packages/ai/CHANGELOG.md', + }, + ], + maintainer: 'uptide-dev', + }, + defaultTarget: '7.0.130', + rules: [ + option( + 'instructions', + 'system', + 'instructions', + CALLS, + '`system` is `instructions` (generateText, streamText, generateObject, streamObject, ToolLoopAgent)', + ), + option( + 'on-end', + 'onFinish', + 'onEnd', + new Set([...TEXT_CALLS, ...UI_STREAMS]), + '`onFinish` is `onEnd` (generateText, streamText, ToolLoopAgent, UI message streams)', + ), + option('on-step-end', 'onStepFinish', 'onStepEnd', CALLS, '`onStepFinish` is `onStepEnd`'), + option( + 'telemetry', + 'experimental_telemetry', + 'telemetry', + TELEMETRY_CALLS, + '`experimental_telemetry` is `telemetry`', + ), + { + id: 'is-step-count', + summary: '`stepCountIs` is `isStepCount`', + severity: 'deprecated', + kinds: [], + symbols: /$^/, + guide: '`stepCountIs(n)` is `isStepCount(n)`: same stop condition.', + detect: (text) => importSites(text, 'stepCountIs'), + rewrite: (text, finding) => replaceAtSite(text, finding, 'stepCountIs', 'isStepCount'), + }, + { + id: 'full-stream', + summary: '`fullStream` on a streamText result is `stream`', + severity: 'deprecated', + kinds: ['deprecated'], + symbols: /#fullStream$/, + guide: + '`result.fullStream` is `result.stream`, the same parts. Rename a destructured `fullStream` and its uses together, unless the scope already has a `stream`.', + rewrite: (text, finding) => + // A destructured local becomes `stream` only where no other `stream` can collide. + declaresName(text, 'stream') + ? onlyOnLine( + text, + finding.usage.line, + /\.fullStream\b/, + '.stream', + '`.fullStream` is `.stream`', + ) + : onlyOnLine( + text, + finding.usage.line, + /\bfullStream\b/, + 'stream', + '`fullStream` is `stream`', + ), + }, + { + id: 'total-usage', + summary: '`totalUsage` is `usage`, which covers every step in AI SDK 7', + severity: 'deprecated', + kinds: ['deprecated'], + symbols: /#totalUsage$/, + guide: + '`result.totalUsage` is `result.usage`: in AI SDK 7 `usage` sums every step, which is what `totalUsage` was.', + rewrite: (text, finding) => + onlyOnLine(text, finding.usage.line, /\.totalUsage\b/, '.usage', '`totalUsage` is `usage`'), + }, + { + id: 'ui-message-stream', + summary: + 'stream result helpers (`toUIMessageStream`, `toUIMessageStreamResponse`, …) are standalone functions', + severity: 'deprecated', + kinds: ['deprecated'], + symbols: + /#(?:toUIMessageStream|toUIMessageStreamResponse|pipeUIMessageStreamToResponse|toTextStreamResponse|pipeTextStreamToResponse)$/, + guide: + '`result.toUIMessageStream(options)` is `toUIMessageStream({ stream: result.stream, ...options })`, imported from "ai"; `result.toUIMessageStreamResponse()` is `createUIMessageStreamResponse({ stream: toUIMessageStream({ stream: result.stream }) })`. Keep every option.', + }, + { + id: 'removed-experimental-option', + summary: + '`experimental_activeTools`, `experimental_prepareStep` and `experimental_output` are gone: `activeTools`, `prepareStep`, `output`', + severity: 'breaking', + kinds: ['type', 'signature'], + symbols: /^TS2353$/, + message: /'experimental_(?:activeTools|prepareStep|output)'/, + guide: + 'The experimental option was promoted: rename `experimental_activeTools` to `activeTools`, `experimental_prepareStep` to `prepareStep`, `experimental_output` to `output`. Nothing else changes.', + rewrite: (text, finding) => { + const name = /'experimental_(activeTools|prepareStep|output)'/.exec( + finding.usage.compileError ?? '', + )?.[1]; + return name + ? onlyOnLine( + text, + finding.usage.line, + new RegExp(`\\bexperimental_${name}\\b`), + name, + `\`experimental_${name}\` is \`${name}\``, + ) + : { text, applied: false, reason: 'not a removed experimental option' }; + }, + }, + { + id: 'telemetry-metadata', + summary: + 'telemetry `metadata` is gone: per-call values move to `runtimeContext`, named in `telemetry.includeRuntimeContext`', + severity: 'breaking', + // The type diff names it (`TelemetrySettings#metadata` removed); the compiler alone says TS2353. + kinds: ['removed', 'type', 'signature'], + symbols: /(?:^TS2353$|#metadata$)/, + message: /'metadata'/, + guide: + 'Move each `metadata` entry to a top-level `runtimeContext: { key: value }` on the call, and list it in `telemetry: { includeRuntimeContext: { key: true } }`. Keep functionId and every other telemetry setting.', + }, + { + id: 'tool-context', + summary: + 'tool execution options carry a typed `context`: `experimental_context` is `context`, and a direct `tool.execute` call passes one', + severity: 'breaking', + kinds: ['type', 'signature', 'required'], + symbols: /^TS\d+$/, + message: /'(?:experimental_)?context'/, + guide: + 'A tool reads `context` (was `experimental_context`) from its execute options; the call site supplies it per tool with `toolsContext`, and shared values with `runtimeContext` (was `context`). A direct `tool.execute(input, { toolCallId, messages })` call adds `context`. Do not cast to make it compile.', + }, + ], + behavior: [ + { + id: 'telemetry-registration', + summary: + 'AI SDK 7 emits no OpenTelemetry spans by itself: register an integration once (`registerTelemetry(new OpenTelemetry())` from "ai" and "@ai-sdk/otel"), or tracing silently stops', + reported: ['decision'], + }, + { + id: 'results-cover-all-steps', + summary: + '`usage`, `content`, `toolCalls`, `toolResults`, `files`, `sources` and `warnings` on a result and on the `onEnd` event now cover every step; the last step alone is `finalStep`', + reported: ['decision'], + }, + { + id: 'system-messages-rejected', + summary: + 'system messages inside `messages` or `prompt` are rejected by default: pass them as `instructions`, or set `allowSystemInMessages`', + reported: ['decision'], + }, + { + id: 'prepare-step-carries-forward', + summary: + 'what `prepareStep` returns for instructions and messages now carries forward to later steps', + reported: ['decision'], + }, + { + id: 'ui-stream-errors-redacted', + summary: + 'UI message streams send "An error occurred." instead of the error: pass `onError` to keep the details', + reported: ['decision'], + }, + { + id: 'companion-packages', + summary: + "`@ai-sdk/react`, `@ai-sdk/provider` and the provider packages move to their AI SDK 7 majors with `ai`; this upgrade bumps `ai` alone, so bump them on the same branch (ai 7 next to @ai-sdk/react 3 mixes two majors' types)", + reported: ['decision'], + }, + { + id: 'esm-only', + summary: 'every `ai` and `@ai-sdk/*` package is ESM-only and needs Node 22 or later', + reported: ['decision'], + }, + ], + instructions: + 'Migrate only the reported site to AI SDK 7, from the compiler error and the guide above. Keep every value, prompt, tool and callback body as it is. Never cast or suppress a diagnostic to make it compile. If the site needs a decision the guide leaves open (which step a value should come from, what a runtime context holds), say so instead of choosing.', +}); diff --git a/packages/core/src/packs/ai/verification.json b/packages/core/src/packs/ai/verification.json new file mode 100644 index 0000000..55a7fee --- /dev/null +++ b/packages/core/src/packs/ai/verification.json @@ -0,0 +1,10 @@ +{ + "status": "verified", + "truth": "sha256-f02a5b04df96c633f6d2430721f47f34ff1777c8ebbe316ce7748fb6f652bce1", + "repos": 2, + "breaking": { + "precision": 1, + "recall": 0.3636, + "falsePositives": 0 + } +} diff --git a/packages/core/src/packs/registry.ts b/packages/core/src/packs/registry.ts index b991804..e40329f 100644 --- a/packages/core/src/packs/registry.ts +++ b/packages/core/src/packs/registry.ts @@ -1,6 +1,8 @@ // Generated by `uptide pack new` (packages/core/src/packs/tooling/registry-file.ts). // Every pack in this tree, with the status `uptide pack test --write` recorded for it. +import { aiPack } from './ai/index.js'; +import aiVerification from './ai/verification.json' with { type: 'json' }; import type { PackVerification, RegisteredPack } from './contract.js'; import { stripePack } from './stripe/index.js'; import stripeVerification from './stripe/verification.json' with { type: 'json' }; @@ -8,6 +10,7 @@ import { zodPack } from './zod/index.js'; import zodVerification from './zod/verification.json' with { type: 'json' }; export const REGISTRY: readonly RegisteredPack[] = [ + { dir: 'ai', pack: aiPack, verification: aiVerification as PackVerification }, { dir: 'stripe', pack: stripePack, verification: stripeVerification as PackVerification }, { dir: 'zod', pack: zodPack, verification: zodVerification as PackVerification }, ]; From 60361ad30c91b86556fd2bcf533675468ee27450 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 17:08:35 +0000 Subject: [PATCH 9/9] feat: upgrade a package together with the packages that must move with it `fix ai` bumped ai alone: ai and the @ai-sdk/* packages pin the same @ai-sdk/provider, so ai 7 next to @ai-sdk/react 3 was an install no real upgrade has, and verification on vercel/chatbot failed with ~45 errors that came from the install, not the code. check now finds the group `list` already draws (family, peer link, shared pin) among the installed dependencies and moves each member to the version that agrees with the target: the version the target pins, else the installed one when it already accepts the target, else its newest release that pins or accepts it, else one whose exact pins agree. The report carries `companions` and `companionConflicts`; check's plan, the fix summary (`With`) and the PR description say which and why. fix bumps and installs them in one step and one commit; the lock guard allows changes inside any of their subtrees only. npm and pnpm resolve the exact version first, so `^4.0.1` keeps the 4.0.1 ai pins. A member with no release that agrees stops fix before it writes anything. Ground-truth entries take `with`, the packages the repository's own upgrade moved; pack test prints what check moves and fails, and the pack is not verified, when check would leave one behind. The ai pack's ground truth records them for vercel/chatbot and miurla/morphic, and its "companion packages" review note is gone. On a copy of vercel/chatbot, `fix ai --target 7.0.9 --no-llm --include-deprecated` now passes verification (1 error after the bump -> 0). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01B2T8GmEVnZwfNrugajEMqi Signed-off-by: Claude --- .changeset/companion-upgrades.md | 12 + docs/architecture.md | 16 ++ docs/cli.md | 5 + docs/packs.md | 8 + packages/cli/src/format-check.test.ts | 33 +++ packages/cli/src/format-check.ts | 13 +- packages/cli/src/format-fix.ts | 5 + packages/cli/src/pack.ts | 3 + packages/core/src/check/check.ts | 134 +++++++++++- packages/core/src/check/companions.test.ts | 151 +++++++++++++ packages/core/src/check/companions.ts | 205 ++++++++++++++++++ packages/core/src/domain/io.ts | 9 + packages/core/src/domain/report.ts | 7 + packages/core/src/errors.ts | 1 + packages/core/src/fetch/npm-fetcher.ts | 6 + packages/core/src/fetch/registry.ts | 33 +++ packages/core/src/fix/generic.test.ts | 61 ++++++ packages/core/src/fix/managers/lock-guard.ts | 16 +- packages/core/src/fix/managers/npm-lock.ts | 12 +- .../core/src/fix/managers/text-lock.test.ts | 52 +++++ packages/core/src/fix/managers/text-lock.ts | 32 ++- packages/core/src/fix/managers/upgrade.ts | 49 +++-- packages/core/src/fix/report.ts | 8 + packages/core/src/fix/run.ts | 32 ++- packages/core/src/fix/types.ts | 2 + packages/core/src/packs/ai/ground-truth.json | 15 +- packages/core/src/packs/ai/index.ts | 6 - packages/core/src/packs/ai/verification.json | 2 +- packages/core/src/packs/contract.ts | 6 + packages/core/src/packs/tooling/pack-test.ts | 33 ++- 30 files changed, 909 insertions(+), 58 deletions(-) create mode 100644 .changeset/companion-upgrades.md create mode 100644 packages/core/src/check/companions.test.ts create mode 100644 packages/core/src/check/companions.ts diff --git a/.changeset/companion-upgrades.md b/.changeset/companion-upgrades.md new file mode 100644 index 0000000..aaa35dc --- /dev/null +++ b/.changeset/companion-upgrades.md @@ -0,0 +1,12 @@ +--- +"uptide": minor +--- + +`check` and `fix` upgrade a package together with the packages that must move with it, +using the groups `list` already draws (family, peer link, shared pin), each at the version +that agrees with the target. `fix ai` also bumps `@ai-sdk/react`, `@ai-sdk/provider` and the +installed `@ai-sdk/*` providers in one install and one commit; check's plan, the fix summary +and the PR description say which and why, and `fix` stops before changing anything when a +member has no release that agrees. npm and pnpm installs resolve the exact version first, +so a `^` range keeps the version the target pins. Ground-truth entries take `with`, the +packages the real upgrade moved, and `pack test` fails when check would leave one behind. diff --git a/docs/architecture.md b/docs/architecture.md index 96cb7ac..87335bb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -77,6 +77,22 @@ never fetched as dependencies of one another), findings per member, and one repo `@aws-sdk/* (3 packages)` with `members` listed. A member with nothing to analyze leaves the group. +### Companions + +A package can also have to move with packages outside its scope (`src/check/companions.ts`). +`ai` and the `@ai-sdk/*` packages pin the same `@ai-sdk/provider`: `ai` 7 next to +`@ai-sdk/react` 3 is an install no real upgrade has, with two copies of the provider's +types. When `check` is asked about one package, the group is the one `list` draws +(`dependencyGroups`: family, peer link, shared pin) among the dependencies installed in its +workspaces, and each member moves to the version that agrees with the target: the version +the target pins exactly; else its newest release that pins or accepts the target; else its +newest release whose exact pins agree with the target's. The report is the lead's +(`ai (3 packages)`) with `companions` (name, from, to, why) and `companionConflicts` (a member +no release agrees with). `fix` installs them together: the version is resolved exactly first +and the range style restored after, so `^4.0.1` stays at the 4.0.1 the target pins; the +lockfile may change inside any of their subtrees and nowhere else. A conflict stops `fix` +before it writes anything. + ### One dependency, one decision A dependency at the same installed version and target in several workspaces is one entry diff --git a/docs/cli.md b/docs/cli.md index 562608f..825d142 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -266,6 +266,11 @@ Everything happens in a temporary clone of your repository, never in your checko with the stored description. 2. **Upgrades.** Bumps the version in every workspace that declares it (and in pnpm catalogs), preserving `^`, `~`, or exact ranges. Installs run in a temporary git worktree with lifecycle scripts disabled. The real package manager produces the lockfile; Uptide rejects changes outside the target dependency subtree before copying it back unchanged. One commit. + Packages that must move with it move in the same install and commit, each at the version + that agrees with the target (see "Companions" in [architecture](./architecture.md)): + `fix ai` also bumps `@ai-sdk/react`, `@ai-sdk/provider` and every installed `@ai-sdk/*` + provider, and `check`'s plan, the summary (`With`) and the PR description say which and + why. When one of them has no release that agrees, `fix` stops before changing anything. 3. **Rule-based fixes.** Deterministic rewrites for the changes it has rules for, such as zod's `required_error` / `invalid_type_error` (add `--include-deprecated` for `z.string().email()`-style chains). One commit. diff --git a/docs/packs.md b/docs/packs.md index ade004a..940244f 100644 --- a/docs/packs.md +++ b/docs/packs.md @@ -142,6 +142,7 @@ Runtime never downloads instructions: they are checked in, with the date they we "directory": "web", "from": "3.25.76", "to": "4.6.5", + "with": { "@scope/companion": "2.0.1" }, "why": "what this repository exercises", "findings": [{ "file": "src/schema.ts", "line": 12, "rule": "error-params" }] } @@ -189,6 +190,13 @@ source has is a false positive, and it stays one. `directory` is the project inside the repository when it is not at the root; `from` is the version its lockfile has at `commit`, and `pack test` says so when it is not. +`with` lists the packages the repository's own upgrade moved together with this one, at the +versions it chose (from its `package.json` at the upgrade commit). `pack test` prints what +`check` moves with the package there and fails when it would leave one of them behind, or +cannot move one consistently: a pack whose `fix` produces an install the real upgrade never +had is not verified. Versions may differ (check picks the release that agrees with `to`); +the names may not. + `fixture` entries (`"fixture": "fixtures/repos/storefront"`) are repositories in this tree. They are scored like the others and never count toward `verified`. diff --git a/packages/cli/src/format-check.test.ts b/packages/cli/src/format-check.test.ts index 0cedfe6..5aa27f7 100644 --- a/packages/cli/src/format-check.test.ts +++ b/packages/cli/src/format-check.test.ts @@ -228,6 +228,39 @@ describe('formatCheck, the first screen', () => { expect(out).toContain('@aws-sdk/* (2 packages) '); }); + it('names what moves with the package, why, and offers fix for the whole group', () => { + const out = formatCheck( + report([ + pkg({ + name: 'ai', + installed: '6.0.116', + latest: '7.0.9', + target: '7.0.9', + majorsBehind: 1, + status: 'breaking', + findings: [removed('app/chat.ts', 3)], + members: [ + { name: 'ai', installed: '6.0.116', target: '7.0.9' }, + { name: '@ai-sdk/react', installed: '3.0.118', target: '4.0.10' }, + ], + companions: [ + { + name: '@ai-sdk/react', + from: '3.0.118', + to: '4.0.10', + reason: '@ai-sdk/react 4.0.10 pins ai 7.0.9', + }, + ], + }), + ]), + { ...shown, fixable: ['ai'] }, + ); + expect(out).toContain( + '↑ upgrades with @ai-sdk/react 3.0.118 → 4.0.10 @ai-sdk/react 4.0.10 pins ai 7.0.9', + ); + expect(out).toContain('npx uptide@next fix ai'); + }); + it('says where each target came from: the npm dist-tag, or the one asked for', () => { const latest = formatCheck(report([pkg({ name: 'twilio' })]), shown); expect(latest).toContain('5.0.1 → 5.3.0 minor · latest on npm'); diff --git a/packages/cli/src/format-check.ts b/packages/cli/src/format-check.ts index 4be7b07..11dddfc 100644 --- a/packages/cli/src/format-check.ts +++ b/packages/cli/src/format-check.ts @@ -281,6 +281,14 @@ function sectionLines(row: Row, colors: Colors): string[] { ` ${colors.yellow('!')} ${plural(sites, 'deprecated call')} (${deprecatedNames(deprecated)}) ${colors.dim(who)}`, ); } + // What moves with it: one install, at versions that agree, the way `fix` upgrades it. + const companions = row.p.companions ?? []; + if (companions.length > 0) + lines.push( + ` ${colors.cyan('↑')} upgrades with ${companions.map((c) => `${c.name} ${c.from} → ${c.to}`).join(', ')} ${colors.dim(companions.map((c) => c.reason).join('; '))}`, + ); + for (const conflict of row.p.companionConflicts ?? []) + lines.push(` ${colors.red('✗')} ${conflict}: no consistent upgrade`); // An importer the manifest does not show, or one the analysis could not reach: the reader // decides whether the sites above are all of them. for (const note of importers) lines.push(` ${colors.yellow('⚠')} ${note}`); @@ -345,8 +353,9 @@ export function nextCommands(rows: Row[], opts: FormatCheckOptions): [string, st opts.fixable === true || (Array.isArray(opts.fixable) && opts.fixable.includes(name)); for (const row of rows) { const name = row.p.name; - // A release group is several packages: fix takes one. - if (!fixable(name) || row.p.members) continue; + // A release group is several packages: fix takes one, unless the group is the package + // and what moves with it, which `fix` upgrades together. + if (!fixable(name) || (row.p.members && !row.p.companions)) continue; if (commands.length >= FIX_SUGGESTIONS) break; const target = opts.repeat?.targets?.[name]; const pinned = target ? ` --target ${quote(target)}` : ''; diff --git a/packages/cli/src/format-fix.ts b/packages/cli/src/format-fix.ts index c293563..9c50d15 100644 --- a/packages/cli/src/format-fix.ts +++ b/packages/cli/src/format-fix.ts @@ -41,6 +41,11 @@ export function formatFixSummary(report: FixReport, opts: FormatFixOptions = {}) ['Tests', cells.tests], ] as const ).map(([label, text]) => ` ${pad(label, 9)} ${plain(text)}`), + ...(report.companions?.length + ? [ + ` ${pad('With', 9)} ${report.companions.map((c) => `${c.name} ${c.from} → ${c.to}`).join(', ')}`, + ] + : []), // Without a pack the reader is told once, here, what stands behind the edits. ...(report.tier === 'generic' && report.sites.some((s) => s.outcome === 'agent') ? [ diff --git a/packages/cli/src/pack.ts b/packages/cli/src/pack.ts index d143796..0a6f90c 100644 --- a/packages/cli/src/pack.ts +++ b/packages/cli/src/pack.ts @@ -97,6 +97,9 @@ export function formatPackTest(report: PackTestReport, color = false): string { `${head} predicted ${r.predicted.length}, expected ${r.expected.length}, false positives ${r.falsePositives.length}, false negatives ${r.falseNegatives.length}`, ); if (r.installedMismatch) out.push(` ${c.yellow('!')} ${r.installedMismatch}`); + if (r.companions?.length) + out.push(` with ${r.companions.map((m) => `${m.name} ${m.from} → ${m.to}`).join(', ')}`); + for (const left of r.leftBehind ?? []) out.push(` ${c.red('✗')} ${left}`); } out.push(''); out.push( diff --git a/packages/core/src/check/check.ts b/packages/core/src/check/check.ts index 914b0d2..c3dc905 100644 --- a/packages/core/src/check/check.ts +++ b/packages/core/src/check/check.ts @@ -35,6 +35,7 @@ import type { MigrationPack } from '../packs/types.js'; import { diffRuntime, probeRuntime } from '../runtime/runtime.js'; import { resetSharedState } from '../shared-state.js'; import { requireFindUsages } from './capabilities.js'; +import { type CompanionPlan, companionsOf, type InstalledDependency } from './companions.js'; import { arbitrateUnchecked } from './file-kind.js'; import { groupName, releaseGroups } from './groups.js'; import { importedByText, importFileCounts } from './importers.js'; @@ -116,6 +117,11 @@ export interface CheckOptions { hardDeadline?: number; /** Set by `check` for its workspace jobs: the dependencies with something to analyze. */ behind?: string[]; + /** + * Set by `check`: for each package asked for by name, what moves with it (its group, at the + * versions that agree with its target). Analyzed as one group led by it, as `fix` upgrades. + */ + companions?: Record; /** * The packs to use instead of the verified ones (`uptide pack test` passes a candidate). * Packs are code, which no worker receives: with this set, workspaces are checked in this @@ -156,7 +162,7 @@ export async function check(opts: CheckOptions): Promise { const root: RepoDir = { dir: opts.cwd }; const workspaces = (await adapter.workspacePackages?.(root)) ?? ['.']; - const ctx: Ctx = { adapter, fetcher: memoizingFetcher(fetcher), cache, findUsages, opts }; + const memo = memoizingFetcher(fetcher); const installedByWorkspace: Record> = {}; const catalogByWorkspace: Record = {}; for (const workspace of workspaces) { @@ -167,6 +173,19 @@ export async function check(opts: CheckOptions): Promise { .filter(([, spec]) => spec.startsWith('catalog:')) .map(([name]) => name); } + // What moves with each package asked for by name, analyzed with it at the versions that + // agree with its target: the upgrade `fix` makes, not one no install ever produces. + const plans = await companionPlans(opts, memo, adapter, workspaces, installedByWorkspace); + const moving = Object.values(plans).flatMap((p) => p.companions); + if (opts.only && moving.length > 0) + opts = { + ...opts, + only: [...new Set([...opts.only, ...moving.map((c) => c.name)])], + targets: { ...Object.fromEntries(moving.map((c) => [c.name, c.to])), ...opts.targets }, + companions: plans, + }; + else if (Object.keys(plans).length > 0) opts = { ...opts, companions: plans }; + const ctx: Ctx = { adapter, fetcher: memo, cache, findUsages, opts }; // Who imports what, whether or not they declare it: the candidates are the packages asked // for, or every package some workspace declares. const candidates = opts.only ?? [ @@ -819,13 +838,25 @@ async function checkWorkspace(ctx: Ctx, job: WorkspaceJob): Promise ({ name, installed: version, dependsOn: dependsOnOf(installedPackageDirOf(adapter, repo, name, version)?.dir), })), ); + // A package asked for by name leads its companions' group, and absorbs any release group + // one of them was in: one overlay with every target linked. + for (const [lead, plan] of Object.entries(opts.companions ?? {})) { + const members = [lead, ...plan.companions.map((c) => c.name)].filter((n) => + analyzable.some(([name]) => name === n), + ); + if (!members.includes(lead) || members.length < 2) continue; + for (const g of groups) + if (g.some((m) => members.includes(m))) + for (const m of g) if (!members.includes(m)) members.push(m); + groups = [...groups.filter((g) => !g.some((m) => members.includes(m))), members]; + } const groupOf = new Map(groups.flatMap((g) => g.map((m) => [m, g] as const))); const done = new Set(); // The analysis is CPU-bound on one thread: six at once finish together, late, and each @@ -955,9 +986,76 @@ interface RunFetcher extends PackageFetcher { * per package, and one extraction per `name@version`, however many workspaces or target * dependency graphs need it. Extracted directories live until the run ends. */ +/** + * For each package `check` was asked about by name, what has to move with it: its group as + * `list` draws it, at the versions that agree with its target (`companions.ts`). From the + * manifests installed in the workspaces that declare it, and one packument per candidate. + */ +async function companionPlans( + opts: CheckOptions, + fetcher: RunFetcher, + adapter: LanguageAdapter, + workspaces: string[], + installedByWorkspace: Record>, +): Promise> { + const plans: Record = {}; + const manifests = fetcher.manifests; + if (!opts.only || !manifests) return plans; + const linked = (version: string) => /^(link|workspace|file):/.test(version); + for (const lead of opts.only) { + const declaring = workspaces.filter((w) => { + const version = installedByWorkspace[w]?.[lead]; + return version !== undefined && !linked(version); + }); + if (declaring.length === 0) continue; + const target = + opts.targets?.[lead] ?? (await fetcher.resolve(lead, 'latest').catch(() => undefined)); + if (!target) continue; + const installed = new Map(); + for (const workspace of declaring) + for (const [name, version] of Object.entries(installedByWorkspace[workspace] ?? {})) { + if (linked(version)) continue; + const known = installed.get(`${name}@${version}`); + if (known) { + known.workspaces.push(workspace); + continue; + } + const dir = installedPackageDirOf( + adapter, + { dir: resolve(opts.cwd, workspace) }, + name, + version, + )?.dir; + let manifest = {}; + try { + if (dir) manifest = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')); + } catch { + // Unreadable: it can still be named in another package's manifest. + } + installed.set(`${name}@${version}`, { name, version, manifest, workspaces: [workspace] }); + } + try { + const plan = await companionsOf({ + name: lead, + target, + installed: [...installed.values()], + manifests: (name) => manifests(name), + }); + if (plan.companions.length > 0 || plan.conflicts.length > 0) plans[lead] = plan; + } catch { + // A registry that cannot answer leaves the package alone, as before. + } + } + return plans; +} + function memoizingFetcher(inner: PackageFetcher): RunFetcher { const resolved = new Map>(); const versions = new Map>(); + const manifests = new Map< + string, + Promise>>> + >(); const fetched = new Map>(); const once = ( map: Map>, @@ -981,6 +1079,12 @@ function memoizingFetcher(inner: PackageFetcher): RunFetcher { once(versions, name, () => inner.versions?.(name) ?? Promise.resolve([])), } : {}), + ...(inner.manifests + ? { + manifests: (name: string) => + once(manifests, name, () => inner.manifests?.(name) ?? Promise.resolve({})), + } + : {}), async dispose() { for (const dir of fetched.values()) { await dir.then((pkg) => releasePackage(inner, pkg)).catch(() => undefined); @@ -1355,7 +1459,12 @@ async function checkGroup( // A member with nothing to analyze (up to date, unused, unanalyzable) leaves the group; a lone package reports as itself. if (prepared.length === 0) return early.length === 1 ? (early[0] as PackageReport) : mergeEarly(workspace, early); - const name = groupName(prepared.map((p) => p.name)); + // Led by the package asked for, when its companions are in the group: it names the report, + // and its versions are the upgrade's. + const plans = ctx.opts.companions ?? {}; + const leader = prepared.find((p) => plans[p.name] !== undefined && prepared.length > 1); + const plan = leader ? plans[leader.name] : undefined; + const name = leader?.name ?? groupName(prepared.map((p) => p.name)); const notes = prepared.flatMap((p) => prepared.length > 1 ? p.notes.map((n) => `${p.name}: ${n}`) : p.notes, ); @@ -1375,9 +1484,12 @@ async function checkGroup( })), } : {}), - installed: installed[0] as string, - latest: (prepared.length === 1 ? prepared[0]?.latest : targets.at(-1)) as string, - target: targets.at(-1) as string, + installed: (leader?.installedVersion ?? installed[0]) as string, + latest: (leader?.latest ?? + (prepared.length === 1 ? prepared[0]?.latest : targets.at(-1))) as string, + target: (leader?.target ?? targets.at(-1)) as string, + ...(plan?.companions.length ? { companions: plan.companions } : {}), + ...(plan?.conflicts.length ? { companionConflicts: plan.conflicts } : {}), majorsBehind: Math.max(...prepared.map((p) => majorsBehind(p.installedVersion, p.target))), findings: [], callSitesChecked: 0, @@ -1791,7 +1903,8 @@ async function checkGroup( findings.push(...arbitrated); // A pack can see what no type diff shows: a client the SDK bump reconfigures at runtime. const pack = packsOf(ctx.opts).find((candidate) => candidate.name === name); - const only = prepared.length === 1 ? prepared[0] : undefined; + // The pack speaks for the package it covers: alone, or leading its companions. + const only = leader ?? (prepared.length === 1 ? prepared[0] : undefined); const packDirs = (p: Prepared) => { const target = fetched.find((f) => f.p === p); return { @@ -1823,7 +1936,12 @@ async function checkGroup( } } // Without a pack, only what the compiler or the runtime probe confirms is called breaking. - const tier = tierOf(packsOf(ctx.opts), name, installed[0] as string, targets.at(-1) as string); + const tier = tierOf( + packsOf(ctx.opts), + name, + (leader?.installedVersion ?? installed[0]) as string, + (leader?.target ?? targets.at(-1)) as string, + ); const evidence = (f: Finding): Finding['evidence'] => { const p = runtimeOf(f); return evidenceOf(f, p?.runtime, p ? loadRootOf(p) : undefined); diff --git a/packages/core/src/check/companions.test.ts b/packages/core/src/check/companions.test.ts new file mode 100644 index 0000000..44210da --- /dev/null +++ b/packages/core/src/check/companions.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it } from 'vitest'; +import { companionsOf, type InstalledDependency } from './companions.js'; + +type Versions = Record< + string, + { dependencies?: Record; peerDependencies?: Record } +>; + +/** The registry as an AI SDK 6 → 7 upgrade sees it: the packages pin the same provider and provider-utils. */ +const registry: Record = { + ai: { + '6.0.116': { + dependencies: { '@ai-sdk/provider': '3.0.3', '@ai-sdk/provider-utils': '4.0.5' }, + }, + '7.0.9': { + dependencies: { '@ai-sdk/provider': '4.0.1', '@ai-sdk/provider-utils': '5.0.1' }, + }, + }, + '@ai-sdk/provider': { '3.0.3': {}, '4.0.1': {}, '4.0.2': {} }, + '@ai-sdk/react': { + '3.0.118': { dependencies: { ai: '6.0.116', '@ai-sdk/provider-utils': '4.0.5' } }, + '4.0.10': { dependencies: { ai: '7.0.9', '@ai-sdk/provider-utils': '5.0.1' } }, + '4.0.11': { dependencies: { ai: '7.0.10', '@ai-sdk/provider-utils': '5.0.2' } }, + }, + '@ai-sdk/openai': { + '3.0.10': { dependencies: { '@ai-sdk/provider': '3.0.3' } }, + '4.0.5': { dependencies: { '@ai-sdk/provider': '4.0.1' } }, + '4.0.6': { dependencies: { '@ai-sdk/provider': '4.0.2' } }, + '5.0.0-beta.1': { dependencies: { '@ai-sdk/provider': '4.0.1' } }, + }, + zod: { '3.25.76': {}, '4.6.5': {} }, +}; +const manifests = async (name: string) => registry[name] ?? {}; +const at = (name: string, version: string, workspaces = ['.']): InstalledDependency => ({ + name, + version, + manifest: { name, version, ...registry[name]?.[version] }, + workspaces, +}); +const repository = [ + at('ai', '6.0.116'), + at('@ai-sdk/provider', '3.0.3'), + at('@ai-sdk/react', '3.0.118'), + at('@ai-sdk/openai', '3.0.10'), + at('zod', '3.25.76'), +]; + +describe('companionsOf', () => { + it('moves every installed @ai-sdk package with ai to the versions that agree with it', async () => { + // Before groups, `fix ai` moved ai alone: ai 7 next to @ai-sdk/react 3 and @ai-sdk/openai 3, + // two copies of @ai-sdk/provider, and an install no lockfile of the real upgrade has. + const plan = await companionsOf({ + name: 'ai', + target: '7.0.9', + installed: repository, + manifests, + }); + expect(plan.conflicts).toEqual([]); + expect(plan.companions).toEqual([ + { + name: '@ai-sdk/openai', + from: '3.0.10', + to: '4.0.5', + reason: '@ai-sdk/openai 4.0.5 pins @ai-sdk/provider 4.0.1, as ai 7.0.9 does', + }, + { + name: '@ai-sdk/provider', + from: '3.0.3', + to: '4.0.1', + reason: 'ai 7.0.9 pins @ai-sdk/provider 4.0.1', + }, + { + name: '@ai-sdk/react', + from: '3.0.118', + to: '4.0.10', + reason: '@ai-sdk/react 4.0.10 pins ai 7.0.9', + }, + ]); + }); + + it('leaves a package with no companion installed alone', async () => { + const plan = await companionsOf({ + name: 'zod', + target: '4.6.5', + installed: repository, + manifests, + }); + expect(plan).toEqual({ companions: [], conflicts: [] }); + }); + + it('reports a member with no release that agrees, instead of moving the package alone', async () => { + const plan = await companionsOf({ + name: 'ai', + target: '7.0.9', + installed: repository, + manifests: async (name) => + name === '@ai-sdk/react' + ? { '3.0.118': registry['@ai-sdk/react']?.['3.0.118'] ?? {} } + : manifests(name), + }); + expect(plan.conflicts).toEqual([ + '@ai-sdk/react 3.0.118 has no release that agrees with ai 7.0.9', + ]); + }); + + it('only groups packages installed in a workspace that has the package', async () => { + const plan = await companionsOf({ + name: 'ai', + target: '7.0.9', + installed: [ + at('ai', '6.0.116', ['apps/web']), + at('@ai-sdk/react', '3.0.118', ['apps/web']), + at('@ai-sdk/openai', '3.0.10', ['apps/docs']), + ], + manifests, + }); + expect(plan.companions.map((c) => c.name)).toEqual(['@ai-sdk/react']); + }); +}); + +describe('companionsOf, a member that already agrees', () => { + it('stays where it is', async () => { + // @modelcontextprotocol/sdk takes zod `^3.25 || ^4.0`: zod 4 needs nothing else to move. + const versions: Versions = { + '1.28.0': { + dependencies: { zod: '^3.25 || ^4.0' }, + peerDependencies: { zod: '^3.25 || ^4.0' }, + }, + '1.32.1': { + dependencies: { zod: '^3.25 || ^4.0' }, + peerDependencies: { zod: '^3.25 || ^4.0' }, + }, + }; + const plan = await companionsOf({ + name: 'zod', + target: '4.6.5', + installed: [ + at('zod', '3.25.76'), + { + name: '@modelcontextprotocol/sdk', + version: '1.28.0', + manifest: { name: '@modelcontextprotocol/sdk', version: '1.28.0', ...versions['1.28.0'] }, + workspaces: ['.'], + }, + ], + manifests: async (name) => + name === '@modelcontextprotocol/sdk' ? versions : manifests(name), + }); + expect(plan).toEqual({ companions: [], conflicts: [] }); + }); +}); diff --git a/packages/core/src/check/companions.ts b/packages/core/src/check/companions.ts new file mode 100644 index 0000000..0dd0be8 --- /dev/null +++ b/packages/core/src/check/companions.ts @@ -0,0 +1,205 @@ +import { satisfies } from '../fetch/range.js'; +import type { Manifest } from '../list/evidence.js'; +import { dependencyGroups } from '../list/groups.js'; +import type { ListedDependency } from '../list/list.js'; +import { compareVersions, parseVersion } from './version.js'; + +/** + * The packages that have to move with one upgrade. `ai` and the `@ai-sdk/*` packages a + * repository installs pin the same `@ai-sdk/provider`: `ai` 7 next to `@ai-sdk/react` 3 is + * an install no lockfile of the upgrade has, with two copies of the provider's types. The + * group is the one `list` shows (`dependencyGroups`: family, peer link, shared pin); the + * version each companion moves to is the one that agrees with the target: + * - the version the target pins exactly (`ai@7.0.9` → `@ai-sdk/provider` 4.0.1); + * - else the installed one, when its range for the target already accepts it + * (`@modelcontextprotocol/sdk` 1.28.0 takes `zod` `^3.25 || ^4.0`: nothing to move); + * - else its newest release that pins the target exactly, or whose range for it accepts + * the target (`@ai-sdk/react@4.0.10` → `ai` 7.0.9); + * - else its newest release whose exact pins agree with every exact pin of the target + * (`@ai-sdk/openai` on `@ai-sdk/provider`). + */ +export interface Companion { + name: string; + from: string; + to: string; + /** Why this version: `ai 7.0.9 pins @ai-sdk/provider 4.0.1`. */ + reason: string; +} + +export interface InstalledDependency { + name: string; + version: string; + /** The installed copy's manifest: what it depends on and pins today. */ + manifest: Manifest; + workspaces: string[]; +} + +export interface CompanionPlan { + companions: Companion[]; + /** Members of the group with no release that agrees: the upgrade cannot be consistent. */ + conflicts: string[]; + /** The group as `list` names it (`@ai-sdk family, shared @ai-sdk/provider-utils`). */ + reason?: string; +} + +type Manifests = Record>; + +const EXACT = /^\d+\.\d+\.\d+(?:-[\w.]+)?$/; +const scopeOf = (name: string): string | undefined => + name.startsWith('@') && !name.startsWith('@types/') ? name.split('/')[0] : undefined; +const stable = (version: string): boolean => { + const parsed = parseVersion(version); + return parsed !== undefined && parsed.pre === undefined; +}; +const accepts = (range: string, version: string): boolean => { + try { + return satisfies(version, range); + } catch { + return false; + } +}; +const exactPins = (m: Pick): Map => + new Map(Object.entries(m.dependencies ?? {}).filter(([, v]) => EXACT.test(v))); + +/** Whether `other` is plausibly in `lead`'s group, from what is installed: worth asking the registry about. */ +function related(lead: InstalledDependency, other: InstalledDependency): boolean { + if (!lead.workspaces.some((w) => other.workspaces.includes(w))) return false; + const mentions = (m: Manifest, name: string) => + name in (m.dependencies ?? {}) || name in (m.peerDependencies ?? {}); + if (mentions(lead.manifest, other.name) || mentions(other.manifest, lead.name)) return true; + const theirs = exactPins(other.manifest); + for (const [dep, version] of exactPins(lead.manifest)) + if (theirs.get(dep) === version) return true; + return false; +} + +/** The release of `name` that agrees with `lead@target`, and why; undefined when none does. */ +function agreeing( + installed: InstalledDependency, + lead: string, + target: string, + targetManifest: Pick, + versions: Manifests, +): { version: string; reason: string } | undefined { + const name = installed.name; + const pinned = targetManifest.dependencies?.[name]; + if (pinned && EXACT.test(pinned) && versions[pinned]) + return { version: pinned, reason: `${lead} ${target} pins ${name} ${pinned}` }; + const current = versions[installed.version] ?? installed.manifest; + const range = current.dependencies?.[lead] ?? current.peerDependencies?.[lead]; + if (range !== undefined && accepts(range, target)) + return { + version: installed.version, + reason: `${name} ${installed.version} accepts ${lead} ${target}`, + }; + const candidates = Object.keys(versions).filter(stable).sort(compareVersions).reverse(); + for (const version of candidates) { + const m = versions[version] ?? {}; + const range = m.dependencies?.[lead] ?? m.peerDependencies?.[lead]; + if (range === undefined || !accepts(range, target)) continue; + return { + version, + reason: EXACT.test(range) + ? `${name} ${version} pins ${lead} ${target}` + : `${name} ${version} accepts ${lead} ${target} (${range})`, + }; + } + const leadPins = exactPins(targetManifest); + for (const version of candidates) { + const pins = exactPins(versions[version] ?? {}); + const shared = [...pins].filter(([dep]) => leadPins.has(dep)); + if (shared.length > 0 && shared.every(([dep, v]) => leadPins.get(dep) === v)) + return { + version, + reason: `${name} ${version} pins ${shared.map(([dep, v]) => `${dep} ${v}`).join(', ')}, as ${lead} ${target} does`, + }; + } + return undefined; +} + +/** + * What moves with `name@target` in a repository, from its installed dependencies and the + * registry's manifests of every version (`manifests`, one cached packument per package). + */ +export async function companionsOf(input: { + name: string; + target: string; + installed: InstalledDependency[]; + manifests: (name: string) => Promise; +}): Promise { + const lead = input.installed.find((d) => d.name === input.name); + if (!lead) return { companions: [], conflicts: [] }; + const targetManifest = (await input.manifests(input.name))[input.target]; + if (!targetManifest) return { companions: [], conflicts: [] }; + // Plausible members: related to the lead, then the rest of their families. + const direct = input.installed.filter((d) => d.name !== lead.name && related(lead, d)); + const families = new Set(direct.map((d) => scopeOf(d.name)).filter(Boolean)); + const scope = scopeOf(lead.name); + if (scope) families.add(scope); + const candidates = input.installed.filter( + (d) => + d.name !== lead.name && + (direct.includes(d) || + (families.has(scopeOf(d.name)) && d.workspaces.some((w) => lead.workspaces.includes(w)))), + ); + if (candidates.length === 0) return { companions: [], conflicts: [] }; + + const moves = new Map(); + for (const c of candidates) + moves.set( + c.name, + agreeing(c, lead.name, input.target, targetManifest, await input.manifests(c.name)), + ); + + // The group, as `list` draws it, with each member at the version that agrees. + const listed = (d: InstalledDependency, latest: string): ListedDependency => + ({ + name: d.name, + current: d.version, + latest, + workspaces: d.workspaces, + classification: 'used', + reasons: [], + // The package being upgraded leads the group, as the one `list` shows used most. + usage: { files: d === lead ? 1 : 0, callSites: 0, references: 0, workspaces: d.workspaces }, + }) as unknown as ListedDependency; + const packages = [ + listed(lead, input.target), + ...candidates.map((c) => listed(c, moves.get(c.name)?.version ?? c.version)), + ]; + const metadata = new Map(input.installed.map((d) => [d.name, [d.manifest]])); + const targets = new Map([[lead.name, targetManifest]]); + for (const c of candidates) { + const move = moves.get(c.name); + const manifest = move && (await input.manifests(c.name))[move.version]; + if (manifest) targets.set(c.name, manifest); + } + const group = dependencyGroups(packages, metadata, targets).find((g) => + g.members.some((m) => m.name === lead.name), + ); + if (!group) return { companions: [], conflicts: [] }; + const companions: Companion[] = []; + const conflicts: string[] = []; + for (const member of group.members) { + if (member.name === lead.name) continue; + const move = moves.get(member.name); + if (!move) { + conflicts.push( + `${member.name} ${member.current} has no release that agrees with ${lead.name} ${input.target}`, + ); + continue; + } + if (move.version === member.current) continue; + companions.push({ + name: member.name, + from: member.current, + to: move.version, + reason: move.reason, + }); + } + return { + companions: companions.sort((a, b) => a.name.localeCompare(b.name)), + conflicts, + ...(group.reason ? { reason: group.reason } : {}), + }; +} diff --git a/packages/core/src/domain/io.ts b/packages/core/src/domain/io.ts index 5e0fc7e..e22abeb 100644 --- a/packages/core/src/domain/io.ts +++ b/packages/core/src/domain/io.ts @@ -19,6 +19,15 @@ export interface PackageFetcher { bin?: string | Record; }>; versions?(name: string): Promise; + /** Every published version's dependencies and peers. Optional: without it no companion moves with a package. */ + manifests?( + name: string, + ): Promise< + Record< + string, + { dependencies?: Record; peerDependencies?: Record } + > + >; /** * Done with a fetched directory. A fetcher that extracts into a persistent cache keeps it; * without this method the caller removes the directory itself. diff --git a/packages/core/src/domain/report.ts b/packages/core/src/domain/report.ts index b9186bf..b86b1dd 100644 --- a/packages/core/src/domain/report.ts +++ b/packages/core/src/domain/report.ts @@ -117,6 +117,13 @@ export interface PackageReport { /** A dependency, or a release group (`@aws-sdk/*`) upgraded together; `members` lists the group. */ name: string; members?: { name: string; installed: string; target: string }[]; + /** + * What moves with the package `check` was asked about (`check/companions.ts`): its group at + * the versions that agree with its target, each with why; `fix` upgrades them together. + */ + companions?: { name: string; from: string; to: string; reason: string }[]; + /** Members of that group with no release that agrees with the target. */ + companionConflicts?: string[]; /** Set when the entry merges several workspaces (`workspace` is then `*`). */ workspaces?: string[]; /** diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.ts index fe2c3fe..924b75c 100644 --- a/packages/core/src/errors.ts +++ b/packages/core/src/errors.ts @@ -91,6 +91,7 @@ export type ErrorCode = | 'NO_PROJECT' | 'TYPESCRIPT_UNAVAILABLE' | 'INSTALL_FAILED' + | 'INCONSISTENT_UPGRADE' | 'PACKAGE_MANAGER_UNAVAILABLE' | 'PACKAGE_MANAGER_VERSION' | 'LOCKFILE_OUT_OF_SCOPE' diff --git a/packages/core/src/fetch/npm-fetcher.ts b/packages/core/src/fetch/npm-fetcher.ts index c9adafb..1fe3c3b 100644 --- a/packages/core/src/fetch/npm-fetcher.ts +++ b/packages/core/src/fetch/npm-fetcher.ts @@ -11,6 +11,7 @@ import { authHeaders, loadRegistryConfig, type RegistryConfig, registryFor } fro import { downloadTarball, type FetchFn, + listManifests, listVersions, manifestUrl, packumentUrl, @@ -176,6 +177,11 @@ export function createNpmFetcher(opts: NpmFetcherOptions = {}): PackageFetcher { listVersions(name, config, fetchFn), ); }, + async manifests(name) { + return remembered([...packagePathSegments(name), 'manifests'], false, () => + listManifests(name, config, fetchFn), + ); + }, async fetch(name, version): Promise { const target = await resolved(name, version); // The tarball is verified on every fetch, cached or not; the extraction is reused only then. diff --git a/packages/core/src/fetch/registry.ts b/packages/core/src/fetch/registry.ts index 610a9e1..586e13f 100644 --- a/packages/core/src/fetch/registry.ts +++ b/packages/core/src/fetch/registry.ts @@ -210,3 +210,36 @@ export async function listVersions( .filter((v) => parseVersion(v) !== undefined) .sort(compareVersions); } + +/** What one version declares, as the abbreviated packument carries it. */ +export interface VersionDependencies { + dependencies?: Record; + peerDependencies?: Record; +} + +/** + * Every published version's dependencies and peer dependencies, from the same abbreviated + * packument `listVersions` reads: one request answers which release of a package pins what. + */ +export async function listManifests( + name: string, + config: RegistryConfig, + fetchFn: FetchFn, +): Promise> { + const registry = registryFor(name, config); + const packument = await getJson( + packumentUrl(name, registry), + config, + fetchFn, + ); + if (!packument) throw new PackageNotFoundError(name, registry); + const out: Record = {}; + for (const [version, manifest] of Object.entries(packument.versions ?? {})) { + if (parseVersion(version) === undefined) continue; + out[version] = { + ...(manifest.dependencies ? { dependencies: manifest.dependencies } : {}), + ...(manifest.peerDependencies ? { peerDependencies: manifest.peerDependencies } : {}), + }; + } + return out; +} diff --git a/packages/core/src/fix/generic.test.ts b/packages/core/src/fix/generic.test.ts index 07b24df..162e50f 100644 --- a/packages/core/src/fix/generic.test.ts +++ b/packages/core/src/fix/generic.test.ts @@ -294,3 +294,64 @@ describe('fix for a dependency without a pack', () => { expect(result.sites).toEqual([]); }, 20000); }); + +describe('fix with companions', () => { + /** `paint` with `brush` installed beside it: check says brush has to move with paint 2. */ + function withBrush(conflicts: string[] = []) { + const fixture = paintFixture(); + const { root, report, services } = fixture; + const manifest = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')); + manifest.dependencies.brush = '1.0.0'; + writeFileSync(join(root, 'package.json'), `${JSON.stringify(manifest, null, 2)}\n`); + git(root, 'commit', '-am', 'brush'); + const lead = report.packages[0] as CheckReport['packages'][number]; + lead.companions = [ + { name: 'brush', from: '1.0.0', to: '2.0.0', reason: 'brush 2.0.0 pins paint 2.0.0' }, + ]; + if (conflicts.length) lead.companionConflicts = conflicts; + const installs: Parameters[1][] = []; + const install = services.install; + services.install = async (dir, upgrade) => { + installs.push(upgrade); + await install(dir, upgrade); + }; + return { ...fixture, installs }; + } + + it('installs the package and every companion in one step, and says so', async () => { + // Moving paint alone would leave brush 1 pinned to paint 1: the install only + // agrees when both move. + const { root, services, fixer, installs } = withBrush(); + const result = await fix({ cwd: root, only: 'paint', fixer: fixer(0.01) }, services); + expect(installs).toHaveLength(1); + expect(installs[0]).toMatchObject({ + name: 'paint', + version: '2.0.0', + also: [{ name: 'brush', version: '2.0.0' }], + }); + const manifest = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')); + expect(manifest.dependencies).toEqual({ paint: '2.0.0', brush: '2.0.0' }); + expect(git(root, 'log', '--format=%s', '-n', '1', 'HEAD~1')).toBe( + 'chore: upgrade paint to 2.0.0 with brush 2.0.0', + ); + expect(result.companions).toEqual([ + { name: 'brush', from: '1.0.0', to: '2.0.0', reason: 'brush 2.0.0 pins paint 2.0.0' }, + ]); + expect(prBody(result)).toContain( + 'Upgraded with paint, so the install stays consistent:\n\n- `brush` 1.0.0 → 2.0.0 (brush 2.0.0 pins paint 2.0.0)', + ); + }, 20000); + + it('changes nothing when a companion has no version that agrees', async () => { + const { root, services, fixer, installs } = withBrush([ + 'brush 1.0.0 has no release that agrees with paint 2.0.0', + ]); + const head = git(root, 'rev-parse', 'HEAD'); + await expect( + fix({ cwd: root, only: 'paint', fixer: fixer(0.01) }, services), + ).rejects.toMatchObject({ code: 'INCONSISTENT_UPGRADE' }); + expect(installs).toEqual([]); + expect(git(root, 'rev-parse', 'HEAD')).toBe(head); + expect(git(root, 'status', '--porcelain')).toBe(''); + }); +}); diff --git a/packages/core/src/fix/managers/lock-guard.ts b/packages/core/src/fix/managers/lock-guard.ts index 7a025fb..ab21bc2 100644 --- a/packages/core/src/fix/managers/lock-guard.ts +++ b/packages/core/src/fix/managers/lock-guard.ts @@ -10,6 +10,10 @@ export interface LockGraph { metadata: unknown; } const sections = ['dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies']; +/** What an upgrade moves: one package, or a package and the companions that move with it. */ +export type Targets = string | readonly string[]; +export const targetNames = (targets: Targets): readonly string[] => + typeof targets === 'string' ? [targets] : targets; function stable(value: unknown): string { if (Array.isArray(value)) return `[${value.map(stable).join(',')}]`; if (value && typeof value === 'object') @@ -21,12 +25,13 @@ function stable(value: unknown): string { } export function withoutTarget( value: Record, - target: string, + target: Targets, ): Record { const result = structuredClone(value); for (const section of sections) { if (result[section] && typeof result[section] === 'object') - delete (result[section] as Record)[target]; + for (const name of targetNames(target)) + delete (result[section] as Record)[name]; } return result; } @@ -37,10 +42,11 @@ export interface LockDiff { allowed: string[]; } /** Compare entries, not lines: formatting changes cannot disguise an unrelated resolution. */ -export function assertLockScope(before: LockGraph, after: LockGraph, target: string): LockDiff { +export function assertLockScope(before: LockGraph, after: LockGraph, target: Targets): LockDiff { + const names = targetNames(target); const allowed = new Set(); for (const graph of [before, after]) { - const queue = [...graph.records].filter(([, r]) => r.name === target).map(([k]) => k); + const queue = [...graph.records].filter(([, r]) => names.includes(r.name)).map(([k]) => k); const visited = new Set(); for (const key of queue) { if (visited.has(key)) continue; @@ -80,7 +86,7 @@ export function assertLockScope(before: LockGraph, after: LockGraph, target: str }); throw new UptideError( 'LOCKFILE_OUT_OF_SCOPE', - `Install changed entries outside ${target}'s dependency subtree: ${described.join('; ')}. No upgrade will be committed.`, + `Install changed entries outside ${names.join(', ')}'s dependency subtree: ${described.join('; ')}. No upgrade will be committed.`, ); } return diff; diff --git a/packages/core/src/fix/managers/npm-lock.ts b/packages/core/src/fix/managers/npm-lock.ts index 2f760b5..2440b68 100644 --- a/packages/core/src/fix/managers/npm-lock.ts +++ b/packages/core/src/fix/managers/npm-lock.ts @@ -1,12 +1,18 @@ import { dirname } from 'node:path'; import { UptideError } from '../../errors.js'; -import { type LockGraph, type LockRecord, withoutTarget } from './lock-guard.js'; +import { + type LockGraph, + type LockRecord, + type Targets, + targetNames, + withoutTarget, +} from './lock-guard.js'; type Entry = Record & { dependencies?: Record; requires?: Record; }; -export function npmGraph(text: string, target: string): LockGraph { +export function npmGraph(text: string, target: Targets): LockGraph { const lock = JSON.parse(text); if (![2, 3].includes(lock.lockfileVersion) || !lock.packages) throw new UptideError( @@ -32,7 +38,7 @@ export function npmGraph(text: string, target: string): LockGraph { : undefined; if (workspace && Object.hasOwn(importers, workspace) && own.requires) { own.requires = { ...own.requires }; - delete own.requires[target]; + for (const name of targetNames(target)) delete own.requires[name]; } nodes.set(key, own); if (dependencies) legacy(dependencies, key); diff --git a/packages/core/src/fix/managers/text-lock.test.ts b/packages/core/src/fix/managers/text-lock.test.ts index 006387d..138719f 100644 --- a/packages/core/src/fix/managers/text-lock.test.ts +++ b/packages/core/src/fix/managers/text-lock.test.ts @@ -28,6 +28,58 @@ snapshots: ).toThrow('outside target'); }); +it('lets a package and its companions move together, and nothing else', () => { + const lock = ( + ai: string, + react: string, + provider: string, + other = '1.0.0', + ) => `lockfileVersion: '9.0' +importers: + .: + dependencies: + ai: + specifier: ${ai} + version: ${ai} + '@ai-sdk/react': + specifier: ${react} + version: ${react}(ai@${ai}) + other: + specifier: ${other} + version: ${other} +packages: + ai@${ai}: {} + '@ai-sdk/react@${react}': {} + '@ai-sdk/provider@${provider}': {} + other@${other}: {} +snapshots: + ai@${ai}: + dependencies: + '@ai-sdk/provider': ${provider} + '@ai-sdk/react@${react}(ai@${ai})': + dependencies: + ai: ${ai} + '@ai-sdk/provider@${provider}': {} + other@${other}: {} +`; + const names = ['ai', '@ai-sdk/react']; + const before = pnpmGraph(lock('6.0.116', '3.0.118', '3.0.3'), names); + expect(() => + assertLockScope(before, pnpmGraph(lock('7.0.9', '4.0.10', '4.0.1'), names), names), + ).not.toThrow(); + // ai alone may not drag @ai-sdk/react to its next major: that is what the group is for. + expect(() => + assertLockScope( + pnpmGraph(lock('6.0.116', '3.0.118', '3.0.3'), 'ai'), + pnpmGraph(lock('7.0.9', '4.0.10', '4.0.1'), 'ai'), + 'ai', + ), + ).toThrow('outside ai'); + expect(() => + assertLockScope(before, pnpmGraph(lock('7.0.9', '4.0.10', '4.0.1', '2.0.0'), names), names), + ).toThrow('outside ai, @ai-sdk/react'); +}); + describe('alias entries (@isaacs/cliui style) survive a manager re-quoting them', () => { const dir = resolve(import.meta.dirname, '../../../../../fixtures/lockfiles/aliases'); const bump = (text: string): string => diff --git a/packages/core/src/fix/managers/text-lock.ts b/packages/core/src/fix/managers/text-lock.ts index 94a8616..45efb15 100644 --- a/packages/core/src/fix/managers/text-lock.ts +++ b/packages/core/src/fix/managers/text-lock.ts @@ -1,5 +1,11 @@ import { UptideError } from '../../errors.js'; -import { type LockGraph, type LockRecord, withoutTarget } from './lock-guard.js'; +import { + type LockGraph, + type LockRecord, + type Targets, + targetNames, + withoutTarget, +} from './lock-guard.js'; interface Block { key: string; @@ -42,7 +48,7 @@ function dependencyPairs(lines: string[], indent: number): [string, string][] { return pairs; } /** Discard only the target dependency block, never text merely mentioning its name. */ -function omitTarget(lines: string[], target: string): string[] { +function omitTarget(lines: string[], target: Targets): string[] { const out: string[] = []; let dependencyIndent = -1, skippedIndent = -1; @@ -55,7 +61,7 @@ function omitTarget(lines: string[], target: string): string[] { else if (indent <= dependencyIndent) dependencyIndent = -1; if (dependencyIndent >= 0 && indent === dependencyIndent + 2) { const key = /^(?:"([^"]+)"|'([^']+)'|([^\s:]+))(?::|\s)/.exec(line.trim()); - if ((key?.[1] ?? key?.[2] ?? key?.[3]) === target) { + if (targetNames(target).includes(key?.[1] ?? key?.[2] ?? key?.[3] ?? '')) { skippedIndent = indent; continue; } @@ -137,7 +143,7 @@ export function parsedEntry(lines: string[], indent: number): Record(); const metadata: Record = {}; const entries = blocks(text.split('\n'), 0); @@ -167,12 +173,16 @@ export function yarnGraph(text: string, target: string): LockGraph { * packages changed: the suffix is the target's version as its dependents see it. It is * compared without the version, so a renamed dependent is the same entry. */ -function withoutTargetPeerVersion(text: string, target: string): string { - const name = target.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - return text.replace(new RegExp(`\\(${name}@[^()]+\\)`, 'g'), `(${target}@*)`); +function withoutTargetPeerVersion(text: string, target: Targets): string { + let out = text; + for (const each of targetNames(target)) { + const name = each.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + out = out.replace(new RegExp(`\\(${name}@[^()]+\\)`, 'g'), `(${each}@*)`); + } + return out; } -export function pnpmGraph(raw: string, target: string): LockGraph { +export function pnpmGraph(raw: string, target: Targets): LockGraph { const text = withoutTargetPeerVersion(raw, target); const records = new Map(); const metadata: string[] = []; @@ -186,7 +196,9 @@ export function pnpmGraph(raw: string, target: string): LockGraph { ...meaningful(lines).filter((line) => { const indent = line.length - line.trimStart().length; if (indent <= 4) - skip = indent === 4 && unquote(line.trim().replace(/:$/, '')) === target; + skip = + indent === 4 && + targetNames(target).includes(unquote(line.trim().replace(/:$/, ''))); return !skip; }), ); @@ -201,7 +213,7 @@ export function pnpmGraph(raw: string, target: string): LockGraph { name, dependencies: [], // A dependent's pointer at the target moves with the target; the rest of it may not. - data: name === target ? data : withoutTarget(data, target), + data: targetNames(target).includes(name) ? data : withoutTarget(data, target), }); if (section === 'snapshots') { const r = records.get(`${section}:${key}`) as LockRecord; diff --git a/packages/core/src/fix/managers/upgrade.ts b/packages/core/src/fix/managers/upgrade.ts index f876193..93fd807 100644 --- a/packages/core/src/fix/managers/upgrade.ts +++ b/packages/core/src/fix/managers/upgrade.ts @@ -4,7 +4,7 @@ import { parseNpm, parsePnpm, parseYarn } from '../../adapters/typescript/lockfi import { UptideError } from '../../errors.js'; import { command } from '../process.js'; import { assertManagerAvailable } from './availability.js'; -import { assertLockScope, type LockDiff, type LockGraph } from './lock-guard.js'; +import { assertLockScope, type LockDiff, type LockGraph, type Targets } from './lock-guard.js'; import { type PackageManager, updateArgs } from './manager.js'; import { npmGraph } from './npm-lock.js'; import { pnpmGraph, yarnGraph } from './text-lock.js'; @@ -13,12 +13,17 @@ export interface Upgrade { version: string; workspaces: string[]; files: string[]; + /** + * What moves with it, each at the version that agrees (`check/companions.ts`): one install, + * and the lockfile may change inside any of their subtrees, never outside them. + */ + also?: { name: string; version: string }[]; } export interface InstallReport extends LockDiff { manager: string; file: string; } -export function lockGraph(pm: PackageManager, text: string, target: string): LockGraph { +export function lockGraph(pm: PackageManager, text: string, target: Targets): LockGraph { return pm.kind === 'npm' ? npmGraph(text, target) : pm.kind === 'pnpm' @@ -41,7 +46,9 @@ export async function upgradeInstall(root: string, upgrade: Upgrade): Promise m.name); + const before = lockGraph(pm, beforeText, names); const originals = new Map( upgrade.files .filter((f) => f.endsWith('package.json')) @@ -53,10 +60,15 @@ export async function upgradeInstall(root: string, upgrade: Upgrade): Promise [m.name, m.version])), + }; writeFileSync(rootManifest, `${JSON.stringify(json, null, 2)}\n`); - } else if (pm.kind === 'npm') { - // Resolve an exact version first, then restore range style; npm reuses this satisfying lock entry. + } else { + // Resolve an exact version first, then restore range style; npm and pnpm reuse this + // satisfying lock entry. A range alone resolves the newest release in it: `^4.0.1` gives + // a second @ai-sdk/provider next to the 4.0.1 that ai pins. for (const [file, text] of originals) { const json = JSON.parse(text); for (const section of [ @@ -65,7 +77,7 @@ export async function upgradeInstall(root: string, upgrade: Upgrade): Promise(); @@ -87,14 +100,18 @@ export async function upgradeInstall(root: string, upgrade: Upgrade): Promise `- \`${c.name}\` ${c.from} → ${c.to} (${c.reason})`), + '', + ); // The note vouches for agent edits; with none, it has nothing to say. if (report.tier === 'generic' && report.sites.some((s) => s.outcome === 'agent')) lines.push(`> ${GENERIC_NOTE(report.package)}`, ''); diff --git a/packages/core/src/fix/run.ts b/packages/core/src/fix/run.ts index a7e93c2..6b6920b 100644 --- a/packages/core/src/fix/run.ts +++ b/packages/core/src/fix/run.ts @@ -222,6 +222,16 @@ async function fixPackage(options: FixOptions, services: FixServices): Promise p.companionConflicts ?? []))]; + if (conflicts.length > 0) + throw new UptideError( + 'INCONSISTENT_UPGRADE', + `${pack.name} ${target} cannot be installed consistently: ${conflicts.join('; ')}. Nothing was changed.`, + ); + const companions = [ + ...new Map(packages.flatMap((p) => p.companions ?? []).map((c) => [c.name, c])).values(), + ].sort((a, b) => a.name.localeCompare(b.name)); const initialContext = { from: [...packages].sort((a, b) => compareVersions(a.installed, b.installed))[0]?.installed ?? '', @@ -285,19 +295,34 @@ async function fixPackage(options: FixOptions, services: FixServices): Promise bumpVersions(root, c.name, c.to)), + ]; + const bump = { + files: [...new Set(bumps.flatMap((b) => b.files))], + workspaces: [...new Set(bumps.flatMap((b) => b.workspaces))], + }; + const moved = companions.map((c) => `${c.name} ${c.to}`).join(', '); const lockfile = await progress( options.onProgress, - { phase: 'install', package: pack.name, detail: `${pack.name} ${target}` }, + { + phase: 'install', + package: pack.name, + detail: `${pack.name} ${target}${moved ? ` with ${moved}` : ''}`, + }, () => services.install(root, { name: pack.name, version: target, workspaces: bump.workspaces, files: bump.files.map((f) => relative(root, f)), + ...(companions.length + ? { also: companions.map((c) => ({ name: c.name, version: c.to })) } + : {}), }), ); - commit(root, `chore: upgrade ${pack.name} to ${target}`, [ + commit(root, `chore: upgrade ${pack.name} to ${target}${moved ? ` with ${moved}` : ''}`, [ ...bump.files, join(root, packageManager(root).lockfile), ]); @@ -454,6 +479,7 @@ async function fixPackage(options: FixOptions, services: FixServices): Promise; why: string; /** Repository-relative `file`, 1-based `line`, and the rule (or behavior note) id. */ findings: GroundTruthFinding[]; diff --git a/packages/core/src/packs/tooling/pack-test.ts b/packages/core/src/packs/tooling/pack-test.ts index c9d4f4a..b0c7361 100644 --- a/packages/core/src/packs/tooling/pack-test.ts +++ b/packages/core/src/packs/tooling/pack-test.ts @@ -2,6 +2,7 @@ import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { createFsSurfaceCache } from '../../cache/fs-surface-cache.js'; import { check } from '../../check/check.js'; +import type { Companion } from '../../check/companions.js'; import type { CheckReport } from '../../domain/report.js'; import { type GroundTruth, @@ -45,6 +46,10 @@ export interface RepoScore { error?: string; /** What the lockfile says is installed, when it is not the ground truth's `from`. */ installedMismatch?: string; + /** What `check` moves with the package there, at versions that agree (`fix` installs them together). */ + companions?: Companion[]; + /** Packages the real upgrade moved (`with`) that `check` would leave behind, or cannot move consistently. */ + leftBehind?: string[]; predicted: ScoredSite[]; expected: ScoredSite[]; /** Predicted sites no expected finding has, at the line level. */ @@ -186,8 +191,24 @@ async function scoreRepo( ]; if (installed.length > 0 && !installed.includes(entry.from)) base.installedMismatch = `the lockfile has ${installed.join(', ')}, the ground truth says ${entry.from}`; + const own = report.packages.filter((p) => p.name === pack.name); + const companions = [ + ...new Map(own.flatMap((p) => p.companions ?? []).map((c) => [c.name, c])).values(), + ].sort((a, b) => a.name.localeCompare(b.name)); + const leftBehind = [ + ...new Set(own.flatMap((p) => p.companionConflicts ?? [])), + ...Object.entries(entry.with ?? {}) + .filter(([name]) => !companions.some((c) => c.name === name)) + .map(([name, version]) => `${name}: the upgrade moved it to ${version}, check leaves it`), + ]; const predicted = predictedSites(report, pack); - return { ...base, predicted, ...scoreSites(predicted, expected) }; + return { + ...base, + ...(companions.length ? { companions } : {}), + ...(leftBehind.length ? { leftBehind } : {}), + predicted, + ...scoreSites(predicted, expected), + }; } catch (err) { return { ...base, error: (err as Error).message }; } @@ -275,8 +296,9 @@ export async function testPack( if (tp + rfp + rfn > 0) rules[id] = tally(tp, rfp, rfn); } + // A pack whose upgrade leaves part of the group behind produces an inconsistent install. const status: PackStatus = - repos.some((r) => r.error) || options.fixturesOnly + repos.some((r) => r.error || r.leftBehind?.length) || options.fixturesOnly ? 'candidate' : statusOf(truth, { falsePositives: breakingFp.length, predicted: breakingPredicted.length }); const verification: PackVerification = { @@ -298,7 +320,12 @@ export async function testPack( writeVerification(options.root, entry.dir, verification); stale = false; } - const passed = scoredAll && fixturesPass(fixtures) && breakingFp.length === 0 && !stale; + const passed = + scoredAll && + fixturesPass(fixtures) && + breakingFp.length === 0 && + !repos.some((r) => r.leftBehind?.length) && + !stale; return { package: pack.name, dir: entry.dir,