From f6400474afdaafdaf4ec6ebc5666d640d98d98b7 Mon Sep 17 00:00:00 2001 From: Lucas Braum Date: Wed, 14 Oct 2026 22:20:54 -0700 Subject: [PATCH 1/4] feat(packs): stricter verified gate: real breaking sites, unexercised rewrites, behavior-only class (UPT-48) verifiedGate replaces statusOf. Besides two public repositories, no breaking false positive and a breaking site found, a verified pack needs five expected breaking sites across public ground truth (fixture repositories never count). An upgrade whose every real expected site is a behavior note's (no rule or compiler site, ioredis 6) is judged on its notes: five note sites, one found, no false positive on a note or a mechanical rule. A mechanical rule with no true positive in public ground truth is reported as unexercised; it does not change the status. pack test prints the class, the site counts and each reason a pack is a candidate, and the computed record carries them. overclaims() is unchanged: a record fails only when it claims verified where the run measures candidate. No verification.json is written here. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lucas Braum --- .changeset/stricter-verified-gate.md | 5 + CONTRIBUTING.md | 8 +- docs/packs.md | 18 ++- packages/cli/src/pack.ts | 11 ++ packages/core/src/index.ts | 4 + packages/core/src/packs/contract.test.ts | 139 ++++++++++++++---- packages/core/src/packs/contract.ts | 95 ++++++++++-- .../core/src/packs/tooling/pack-test.test.ts | 78 +++++++++- packages/core/src/packs/tooling/pack-test.ts | 75 +++++++++- 9 files changed, 381 insertions(+), 52 deletions(-) create mode 100644 .changeset/stricter-verified-gate.md diff --git a/.changeset/stricter-verified-gate.md b/.changeset/stricter-verified-gate.md new file mode 100644 index 0000000..e26f4fd --- /dev/null +++ b/.changeset/stricter-verified-gate.md @@ -0,0 +1,5 @@ +--- +"uptide": minor +--- + +`uptide pack test` applies a stricter verified gate: at least five expected breaking sites across the public ground-truth repositories, besides two repositories and no breaking false positive. Upgrades the compiler cannot see are gated on their behavior notes instead. Mechanical rules no real site exercises are listed as unexercised. The output shows the gate's class, site counts and every reason a pack is a candidate. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f123152..177eede 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,9 +94,11 @@ Rules for a pack change: by `uptide pack test`. Expected findings come from the repository's own upgrade and the compiler, never from Uptide's output. No private code, paths or names anywhere: fixtures are synthetic, ground truth is public. -4. A pack is verified only with ground truth from two public repositories and no false - positive among its breaking findings; until then it ships as a candidate, and the CLI - treats the dependency as generic. +4. A pack is verified only with ground truth from two public repositories, at least five + expected breaking sites across them, and no false positive among its breaking findings + (docs/packs.md has the behavior-only variant); until then it ships as a candidate, and + the CLI treats the dependency as generic. A rewrite no real site exercises is listed + as unexercised: give each mechanical rule at least one real site. 5. Server API changes (Stripe API versions) are never rewritten by rule. ## Write a pack diff --git a/docs/packs.md b/docs/packs.md index d3ffa39..dd46ed7 100644 --- a/docs/packs.md +++ b/docs/packs.md @@ -297,9 +297,21 @@ measures verified`; one whose numbers merely differ passes with a note (`--write ## Verified or candidate -A pack is **verified** only with ground truth from at least two public repositories and no -false positive among its breaking findings there (precision 100%), with at least one -breaking finding predicted. Anything else is a **candidate**: +A pack is **verified** only with ground truth from at least two public repositories, no +false positive among its breaking findings there (precision 100%), at least one breaking +finding predicted, and at least five expected breaking sites across the public repositories +(fixture repositories never count). An upgrade the compiler cannot see, where every +expected site is a behavior note's (ioredis 6: new runtime defaults), is judged on its notes +instead, whatever their severity: at least five expected note sites, one found, and no false positive on a note or a +mechanical rule. `pack test` prints the gate's class, the site counts and every reason a +pack is a candidate. + +A rule with a `rewrite` that no public repository's ground truth exercises (no true +positive) is **unexercised**: `pack test` lists it and records it in `verification.json` +(`unexercised`), because nothing real confirms its rewrite yet. Its sites are meant to run as +assisted; detection is unaffected, and so is the pack's status. + +Anything else is a **candidate**: - it ships in the tree, and `pack test` scores it; - by default, `check`, `list` and `fix` treat the dependency as generic: no `verified` label, the diff --git a/packages/cli/src/pack.ts b/packages/cli/src/pack.ts index 7baeb8c..400f35c 100644 --- a/packages/cli/src/pack.ts +++ b/packages/cli/src/pack.ts @@ -159,6 +159,17 @@ export function formatPackTest(report: PackTestReport, color = false): string { ); } out.push(''); + if (!report.fixturesOnly && report.repos.length > 0) { + const g = report.gate; + out.push( + `gate ${g.class === 'behavior' ? 'behavior-only, ' : ''}${g.breakingSites} breaking site${g.breakingSites === 1 ? '' : 's'}, ${g.behaviorSites} behavior-note site${g.behaviorSites === 1 ? '' : 's'} in public ground truth · ${g.status}`, + ); + for (const reason of g.reasons) out.push(` ${c.yellow('·')} ${reason}`); + if (g.unexercised.length) + out.push( + ` ${c.yellow('·')} no real site exercises ${g.unexercised.join(', ')}: their rewrites run as assisted`, + ); + } if (report.overclaimed) out.push( `${c.red('✗')} verification.json records verified; this run measures candidate: \`uptide pack test ${report.package} --write\` records it`, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f6bbbb6..0cfab5b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -177,6 +177,7 @@ export * from './onboarding/node.js'; export { type BehaviorNote, definePack, + type GateResult, type GroundTruth, type GroundTruthFinding, type GroundTruthRepo, @@ -188,8 +189,11 @@ export { type RegisteredPack, rateLabel, replaceAtSite, + VERIFIED_MIN_BEHAVIOR_SITES, + VERIFIED_MIN_BREAKING_SITES, VERIFIED_MIN_REPOS, type VerificationTally, + verifiedGate, } from './packs/contract.js'; export { activePacks, packStatus, registeredPacks } from './packs/index.js'; export { leadName, leadNames, packLeading } from './packs/leads.js'; diff --git a/packages/core/src/packs/contract.test.ts b/packages/core/src/packs/contract.test.ts index 4fd3403..a01b01f 100644 --- a/packages/core/src/packs/contract.test.ts +++ b/packages/core/src/packs/contract.test.ts @@ -6,11 +6,14 @@ import type { Finding } from '../domain/report.js'; import { companionProblems, definePack, + type GateEvidence, type GroundTruth, recordedStatus, replaceAtSite, - statusOf, truthDigest, + VERIFIED_MIN_BEHAVIOR_SITES, + VERIFIED_MIN_BREAKING_SITES, + verifiedGate, } from './contract.js'; import { activePacks, registeredPacks } from './index.js'; import { runFixtures } from './tooling/fixtures.js'; @@ -85,32 +88,118 @@ describe('every registered pack meets the contract', () => { }); describe('the verified gate', () => { - const truth = (repos: number, fixtures = 0): GroundTruth => ({ - package: 'x', - repos: [ - ...Array.from({ length: repos }, (_, i) => ({ - repo: `o/r${i}`, - commit: 'a'.repeat(40), - from: '1.0.0', - to: '2.0.0', - why: '', - findings: [], - })), - ...Array.from({ length: fixtures }, () => ({ - fixture: 'fixtures/repos/x', - from: '1.0.0', - to: '2.0.0', - why: '', - findings: [], - })), - ], + const evidence = (over: Partial = {}): GateEvidence => ({ + repos: 2, + ruleSites: 5, + breakingSites: 5, + behaviorSites: 0, + breaking: { truePositives: 4, falsePositives: 0 }, + behavior: { truePositives: 0, falsePositives: 0 }, + mechanical: { rename: { truePositives: 3, falsePositives: 0 } }, + ...over, + }); + + it('verifies two public repositories, five breaking sites, none falsely predicted, some found', () => { + expect(verifiedGate(evidence())).toEqual({ + status: 'verified', + class: 'breaking', + reasons: [], + unexercised: [], + }); + }); + + it('needs two public repositories', () => { + expect(verifiedGate(evidence({ repos: 1 })).reasons).toEqual([ + '1 public repository, verified needs 2', + ]); + }); + + it('needs no false positive among breaking findings', () => { + const gate = verifiedGate(evidence({ breaking: { truePositives: 4, falsePositives: 1 } })); + expect(gate.status).toBe('candidate'); + expect(gate.reasons).toEqual(['1 false positive among breaking findings']); + }); + + it(`needs ${VERIFIED_MIN_BREAKING_SITES} breaking sites in public ground truth`, () => { + expect(verifiedGate(evidence({ breakingSites: 4 })).reasons).toEqual([ + '4 breaking sites in public ground truth, verified needs 5', + ]); + expect(verifiedGate(evidence({ breakingSites: 1 })).status).toBe('candidate'); + }); + + it('needs a breaking site found', () => { + expect( + verifiedGate(evidence({ breaking: { truePositives: 0, falsePositives: 0 } })).reasons, + ).toEqual(['no breaking site found']); }); - it('needs two public repositories, no breaking false positive, and something found', () => { - expect(statusOf(truth(2), { falsePositives: 0, predicted: 3 })).toBe('verified'); - expect(statusOf(truth(1, 3), { falsePositives: 0, predicted: 3 })).toBe('candidate'); - expect(statusOf(truth(3), { falsePositives: 1, predicted: 3 })).toBe('candidate'); - expect(statusOf(truth(2), { falsePositives: 0, predicted: 0 })).toBe('candidate'); + + it('turns a mechanical rule no real site confirmed into assisted, without demoting the pack', () => { + const gate = verifiedGate( + evidence({ + mechanical: { + rename: { truePositives: 3, falsePositives: 0 }, + json: { truePositives: 0, falsePositives: 0 }, + defer: { truePositives: 0, falsePositives: 0 }, + }, + }), + ); + expect(gate.status).toBe('verified'); + expect(gate.unexercised).toEqual(['defer', 'json']); }); + + describe("a behavior-only upgrade (every expected site a note's)", () => { + const behavior = (over: Partial = {}): GateEvidence => + evidence({ + ruleSites: 0, + breakingSites: 2, + behaviorSites: 6, + breaking: { truePositives: 0, falsePositives: 0 }, + behavior: { truePositives: 5, falsePositives: 0 }, + mechanical: {}, + ...over, + }); + + it('is verified on its notes: sites, no false positive, something found', () => { + expect(verifiedGate(behavior())).toEqual({ + status: 'verified', + class: 'behavior', + reasons: [], + unexercised: [], + }); + }); + + it(`needs ${VERIFIED_MIN_BEHAVIOR_SITES} behavior-note sites and one found`, () => { + expect( + verifiedGate( + behavior({ behaviorSites: 2, behavior: { truePositives: 0, falsePositives: 0 } }), + ).reasons, + ).toEqual([ + '2 behavior-note sites in public ground truth, verified needs 5', + 'no behavior-note site found', + ]); + }); + + it('needs no false positive on a note or a mechanical rule', () => { + const gate = verifiedGate( + behavior({ + behavior: { truePositives: 5, falsePositives: 2 }, + mechanical: { 'pin-protocol': { truePositives: 1, falsePositives: 1 } }, + }), + ); + expect(gate.class).toBe('behavior'); + expect(gate.reasons).toEqual([ + '2 false positives among behavior-note sites', + '1 false positive among mechanical rules', + ]); + }); + + it('is never the class of a pack with an expected site under a rule', () => { + const gate = verifiedGate(behavior({ ruleSites: 1 })); + expect(gate.class).toBe('breaking'); + expect(gate.status).toBe('candidate'); + }); + }); + it('a record that contradicts itself is a candidate', () => { const record = { status: 'verified' as const, diff --git a/packages/core/src/packs/contract.ts b/packages/core/src/packs/contract.ts index e7e89c6..3fc0449 100644 --- a/packages/core/src/packs/contract.ts +++ b/packages/core/src/packs/contract.ts @@ -160,6 +160,13 @@ export interface PackVerification { breaking: VerificationTally; /** Sites of behavior notes, whatever their severity. Absent from records written before it. */ behavior?: VerificationTally; + /** The gate's class (`verifiedGate`). Absent from records written before UPT-48. */ + class?: 'breaking' | 'behavior'; + /** Expected breaking and behavior-note sites across public repositories. */ + breakingSites?: number; + behaviorSites?: number; + /** Mechanical rules no real site confirmed: `fix` runs their sites as assisted. */ + unexercised?: string[]; } /** @@ -193,8 +200,8 @@ export function rateLabel(tally: VerificationTally, rate: 'precision' | 'recall' } /** - * `verified`: ground truth from at least two public repositories and no false positive among - * its breaking findings there. Anything else is a `candidate`: it ships, `uptide pack test` + * `verified`: what `verifiedGate` says: ground truth from at least two public repositories, no + * false positive among its breaking findings there, and enough real sites to trust it. Anything else is a `candidate`: it ships, `uptide pack test` * scores it, and `check`, `list` and `fix` treat the dependency as generic. */ export type PackStatus = 'verified' | 'candidate'; @@ -205,16 +212,80 @@ export function publicRepos(truth: GroundTruth): GroundTruthRepo[] { return truth.repos.filter((r) => r.repo !== undefined && r.commit !== undefined); } -/** The gate, from what `pack test` measured. */ -export function statusOf( - truth: GroundTruth, - breaking: { falsePositives: number; predicted: number }, -): PackStatus { - return publicRepos(truth).length >= VERIFIED_MIN_REPOS && - breaking.falsePositives === 0 && - breaking.predicted > 0 - ? 'verified' - : 'candidate'; +/** Expected breaking sites, across public repositories, a `verified` pack needs (UPT-48). */ +export const VERIFIED_MIN_BREAKING_SITES = 5; + +/** Expected behavior-note sites, across public repositories, a behavior-only pack needs. */ +export const VERIFIED_MIN_BEHAVIOR_SITES = 5; + +/** + * What the gate reads, all of it from public repositories but the false positives, which count + * wherever `pack test` scored them. `mechanical` is every rule with a `rewrite`, with its real + * true and false positives. + */ +export interface GateEvidence { + repos: number; + /** Expected sites under a rule or `generic`: what the compiler or a rule sees, not a note. */ + ruleSites: number; + breakingSites: number; + behaviorSites: number; + breaking: { truePositives: number; falsePositives: number }; + behavior: { truePositives: number; falsePositives: number }; + mechanical: Record; +} + +/** + * The gate's verdict. `class` is `behavior` for an upgrade the compiler cannot see (every + * expected site is a behavior note's, whatever its severity: ioredis 6), judged on its notes; + * otherwise `breaking`. `reasons` say why a candidate is one. `unexercised` are mechanical rules + * no real site ever confirmed: their sites run as assisted, whatever the status. + */ +export interface GateResult { + status: PackStatus; + class: 'breaking' | 'behavior'; + reasons: string[]; + unexercised: string[]; +} + +const sites = (n: number, what: string): string => `${n} ${what} site${n === 1 ? '' : 's'}`; + +/** The verified gate, from what `pack test` measured. */ +export function verifiedGate(e: GateEvidence): GateResult { + const cls = e.ruleSites === 0 && e.behaviorSites > 0 ? 'behavior' : 'breaking'; + const reasons: string[] = []; + if (e.repos < VERIFIED_MIN_REPOS) + reasons.push( + `${e.repos} public repositor${e.repos === 1 ? 'y' : 'ies'}, verified needs ${VERIFIED_MIN_REPOS}`, + ); + if (e.breaking.falsePositives > 0) + reasons.push( + `${e.breaking.falsePositives} false positive${e.breaking.falsePositives === 1 ? '' : 's'} among breaking findings`, + ); + if (cls === 'breaking') { + if (e.breakingSites < VERIFIED_MIN_BREAKING_SITES) + reasons.push( + `${sites(e.breakingSites, 'breaking')} in public ground truth, verified needs ${VERIFIED_MIN_BREAKING_SITES}`, + ); + if (e.breaking.truePositives === 0) reasons.push('no breaking site found'); + } else { + if (e.behavior.falsePositives > 0) + reasons.push( + `${e.behavior.falsePositives} false positive${e.behavior.falsePositives === 1 ? '' : 's'} among behavior-note sites`, + ); + const ruleFp = Object.values(e.mechanical).reduce((n, r) => n + r.falsePositives, 0); + if (ruleFp > 0) + reasons.push(`${ruleFp} false positive${ruleFp === 1 ? '' : 's'} among mechanical rules`); + if (e.behaviorSites < VERIFIED_MIN_BEHAVIOR_SITES) + reasons.push( + `${sites(e.behaviorSites, 'behavior-note')} in public ground truth, verified needs ${VERIFIED_MIN_BEHAVIOR_SITES}`, + ); + if (e.behavior.truePositives === 0) reasons.push('no behavior-note site found'); + } + const unexercised = Object.entries(e.mechanical) + .filter(([, r]) => r.truePositives === 0) + .map(([id]) => id) + .sort(); + return { status: reasons.length ? 'candidate' : 'verified', class: cls, reasons, unexercised }; } /** The digest a verification record is tied to: the ground-truth file as committed. */ diff --git a/packages/core/src/packs/tooling/pack-test.test.ts b/packages/core/src/packs/tooling/pack-test.test.ts index 16a85c6..f00035e 100644 --- a/packages/core/src/packs/tooling/pack-test.test.ts +++ b/packages/core/src/packs/tooling/pack-test.test.ts @@ -5,7 +5,14 @@ import type { CheckReport, Finding } from '../../domain/report.js'; import { aiPack } from '../ai/index.js'; import type { PackVerification } from '../contract.js'; import { registeredPacks } from '../index.js'; -import { companionOutcome, overclaims, predictedSites, testPack } from './pack-test.js'; +import { + companionOutcome, + gateOf, + overclaims, + predictedSites, + type RepoScore, + testPack, +} from './pack-test.js'; const root = fileURLToPath(new URL('../../../../../', import.meta.url)); const verification = JSON.parse( @@ -176,3 +183,72 @@ it('never finds a fixtures-only run over-claiming, whatever the record says', as ); expect(report).toMatchObject({ recorded: 'verified', status: 'candidate', overclaimed: false }); }); + +it('gates on real sites only: a fixture repository scores, never counts, and never exercises a rule', () => { + const pack = { + rules: [ + { + id: 'rename', + summary: '', + severity: 'breaking' as const, + rewrite: () => ({ applied: false as const, reason: '' }), + }, + { + id: 'json', + summary: '', + severity: 'breaking' as const, + rewrite: () => ({ applied: false as const, reason: '' }), + }, + { id: 'look', summary: '', severity: 'breaking' as const }, + ], + behavior: [], + } as unknown as Parameters[0]; + const site = (file: string, rule: string) => ({ + file, + line: 1, + rule, + severity: 'breaking' as const, + }); + const repo = ( + label: string, + sites: ReturnType[], + over: Partial = {}, + ): RepoScore => ({ + label, + from: '1.0.0', + to: '2.0.0', + predicted: sites, + expected: sites, + falsePositives: [], + falseNegatives: [], + wrongRule: [], + ...over, + }); + const real = (i: number) => + repo(`o/r${i}`, [site('a.ts', 'rename'), site('b.ts', 'look')], { repo: `o/r${i}` }); + const truth = { + package: 'x', + repos: [0, 1].map((i) => ({ + repo: `o/r${i}`, + commit: 'c', + from: '1.0.0', + to: '2.0.0', + why: '', + findings: [], + })), + }; + const fixture = repo('fixture', [site('c.ts', 'json'), site('d.ts', 'json')], { + fixture: 'fixtures/repos/x', + }); + + const gate = gateOf(pack, truth, [real(0), real(1), fixture]); + expect(gate.breakingSites).toBe(4); + expect(gate.unexercised).toEqual(['json']); + expect(gate.reasons).toEqual(['4 breaking sites in public ground truth, verified needs 5']); + + const third = repo('o/r2', [site('e.ts', 'rename')], { repo: 'o/r2' }); + expect(gateOf(pack, truth, [real(0), real(1), third, fixture]).status).toBe('verified'); + // A repository that could not be scored is no evidence. + const broken = { ...third, error: 'clone failed' }; + expect(gateOf(pack, truth, [real(0), real(1), broken]).breakingSites).toBe(4); +}); diff --git a/packages/core/src/packs/tooling/pack-test.ts b/packages/core/src/packs/tooling/pack-test.ts index c0edb09..a5f322d 100644 --- a/packages/core/src/packs/tooling/pack-test.ts +++ b/packages/core/src/packs/tooling/pack-test.ts @@ -6,6 +6,7 @@ import type { Companion } from '../../check/companions.js'; import type { CheckReport } from '../../domain/report.js'; import { companionProblems, + type GateResult, type GroundTruth, type GroundTruthRepo, type Pack, @@ -14,9 +15,9 @@ import { publicRepos, type RegisteredPack, recordedStatus, - statusOf, truthDigest, verificationTally, + verifiedGate, } from '../contract.js'; import { type FixtureResult, fixturesPass, runFixtures } from './fixtures.js'; import { type EnsureOptions, ensureFixture, ensureRepo, label, truthProblems } from './truth.js'; @@ -97,6 +98,8 @@ export interface PackTestReport { breaking: Tally; /** Ground truth, at the line level, the sites of behavior notes, whatever their severity. */ behavior: Tally; + /** The verified gate's verdict, and the real-site counts it read. */ + gate: GateResult & { breakingSites: number; behaviorSites: number }; passed: boolean; } @@ -422,20 +425,26 @@ export async function testPack( repos.filter((r) => !r.error), ); + const gate = gateOf(pack, truth, repos); // A pack whose upgrade leaves part of the group behind produces an inconsistent install. - const status: PackStatus = - repos.some((r) => r.error || r.leftBehind?.length) || options.fixturesOnly - ? 'candidate' - : statusOf(truth, { - falsePositives: breaking.falsePositives, - predicted: breaking.truePositives + breaking.falsePositives, - }); + if (options.fixturesOnly) gate.reasons.unshift('ground truth not scored (--fixtures-only)'); + for (const r of repos) { + if (r.error) gate.reasons.unshift(`${r.label} could not be scored`); + if (r.leftBehind?.length) + gate.reasons.unshift(`${r.label}: check leaves part of the upgrade behind`); + } + const status: PackStatus = gate.reasons.length ? 'candidate' : 'verified'; + gate.status = status; const verification: PackVerification = { status, truth: read ? truthDigest(read.text) : '', repos: publicRepos(truth).length, breaking: verificationTally(breaking), behavior: verificationTally(behavior), + class: gate.class, + breakingSites: gate.breakingSites, + behaviorSites: gate.behaviorSites, + unexercised: gate.unexercised, }; const recorded = recordedStatus(entry.verification); // Fixtures alone say nothing about ground truth: only a full run can find the record stale. @@ -470,10 +479,52 @@ export async function testPack( overall, breaking, behavior, + gate, passed, }; } +/** + * The gate over one run: real sites are those of public repositories (a fixture repository + * scores, never counts), and so are the true positives that exercise a mechanical rule. + */ +export function gateOf( + pack: Pick, + truth: GroundTruth, + repos: RepoScore[], +): PackTestReport['gate'] { + const scored = repos.filter((r) => !r.error); + const real = scored.filter((r) => !r.fixture); + const notes = new Set(pack.behavior.map((b) => b.id)); + const expected = real.flatMap((r) => r.expected); + const breakingSites = expected.filter((e) => siteSeverity(pack, e.rule) === 'breaking').length; + const behaviorSites = expected.filter((e) => notes.has(e.rule)).length; + const ruleSites = expected.length - behaviorSites; + const all = scoreTruth(pack, scored); + const { rules } = scoreTruth(pack, real); + const mechanical = Object.fromEntries( + pack.rules + .filter((r) => r.rewrite) + .map((r) => [ + r.id, + { + truePositives: rules[r.id]?.truePositives ?? 0, + falsePositives: all.rules[r.id]?.falsePositives ?? 0, + }, + ]), + ); + const gate = verifiedGate({ + repos: publicRepos(truth).length, + ruleSites, + breakingSites, + behaviorSites, + breaking: all.breaking, + behavior: all.behavior, + mechanical, + }); + return { ...gate, breakingSites, behaviorSites }; +} + /** * Whether a record claims more than the evidence supports: verified where the run measures * candidate. A record may claim less (candidate where the run measures verified): the runtime @@ -519,6 +570,14 @@ export function unfinishedPack(entry: RegisteredPack, reason: string): PackTestR overall: none, breaking: none, behavior: none, + gate: { + status: 'candidate', + class: 'breaking', + reasons: [reason], + unexercised: [], + breakingSites: 0, + behaviorSites: 0, + }, passed: false, }; } From a65e6c023946656e40fecc62fa60d94d71ad4067 Mon Sep 17 00:00:00 2001 From: Lucas Braum Date: Wed, 14 Oct 2026 23:10:00 -0700 Subject: [PATCH 2/4] feat(packs): run unexercised rules as assisted; per-pack breaking-site minimum (UPT-48) activePacks wraps each pack with the unexercised rules its verification.json records: transform refuses their edits (fix hands the site to the agent with the guide) and check demotes their sites to assisted. The wrapper checks the rule of the applied edit, so hand-written transforms (zod, stripe) are covered; stripe lists the rules its transform edits (meta-free `mechanical`) so the gate counts them as mechanical. meta.minBreakingSites lowers the breaking-site minimum for one pack, and pack test refuses it without a reason under '## Breaking-site minimum' in its RESEARCH.md. The README packs table gains a breaking-sites column. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lucas Braum --- packages/core/src/check/check.ts | 8 ++- packages/core/src/packs/contract.test.ts | 28 +++++++++++ packages/core/src/packs/contract.ts | 18 ++++++- packages/core/src/packs/fixability.ts | 50 ++++++++++++++++++- packages/core/src/packs/index.ts | 3 +- packages/core/src/packs/stripe/index.ts | 2 + .../core/src/packs/tooling/pack-test.test.ts | 26 +++++++++- packages/core/src/packs/tooling/pack-test.ts | 41 +++++++++++---- packages/core/src/packs/types.ts | 5 ++ scripts/docs-packs.ts | 6 +-- 10 files changed, 167 insertions(+), 20 deletions(-) diff --git a/packages/core/src/check/check.ts b/packages/core/src/check/check.ts index f7d2654..d2387c4 100644 --- a/packages/core/src/check/check.ts +++ b/packages/core/src/check/check.ts @@ -434,7 +434,13 @@ export async function check(opts: CheckOptions): Promise { for (const report of results.flat()) { // Packs whose rules carry their own rewrite (zod, and every pack built on the contract). const pack = packsOf(opts).find((p) => p.name === report.name); - if (!pack || !(pack.rules as readonly { rewrite?: unknown }[]).some((r) => r.rewrite)) + if ( + !pack || + !( + (pack.rules as readonly { rewrite?: unknown }[]).some((r) => r.rewrite) || + pack.unexercised?.length + ) + ) continue; const sources = new Map(); report.findings = report.findings.map((finding) => { diff --git a/packages/core/src/packs/contract.test.ts b/packages/core/src/packs/contract.test.ts index a01b01f..df171a2 100644 --- a/packages/core/src/packs/contract.test.ts +++ b/packages/core/src/packs/contract.test.ts @@ -15,6 +15,7 @@ import { VERIFIED_MIN_BREAKING_SITES, verifiedGate, } from './contract.js'; +import { packFixability, withUnexercised } from './fixability.js'; import { activePacks, registeredPacks } from './index.js'; import { runFixtures } from './tooling/fixtures.js'; import { scoreSites } from './tooling/pack-test.js'; @@ -127,6 +128,15 @@ describe('the verified gate', () => { expect(verifiedGate(evidence({ breakingSites: 1 })).status).toBe('candidate'); }); + it("reads the pack's own breaking-site minimum", () => { + expect(verifiedGate(evidence({ breakingSites: 3, minBreakingSites: 3 })).status).toBe( + 'verified', + ); + expect(verifiedGate(evidence({ breakingSites: 2, minBreakingSites: 3 })).reasons).toEqual([ + '2 breaking sites in public ground truth, verified needs 3', + ]); + }); + it('needs a breaking site found', () => { expect( verifiedGate(evidence({ breaking: { truePositives: 0, falsePositives: 0 } })).reasons, @@ -321,6 +331,24 @@ describe('definePack', () => { ).toEqual(companions); }); + it('runs an unexercised rule as assisted: no edit, the site demoted, detection unchanged', () => { + const context = { from: '1.2.0', to: '2.0.0', includeDeprecated: false }; + const text = 'oldName(1);'; + expect(withUnexercised(pack, [])).toBe(pack); + const assisted = withUnexercised(pack, ['rename']); + expect(assisted.transform(text, finding(1, 1), context)).toMatchObject({ + applied: false, + text, + reason: expect.stringContaining('rename rewrite'), + }); + expect(packFixability(finding(1, 1), text, assisted).fixability).toBe('assisted'); + expect(packFixability(finding(1, 1), text, pack).fixability).toBe('mechanical'); + // Another rule's edits are untouched. + expect(withUnexercised(pack, ['other']).transform(text, finding(1, 1), context).applied).toBe( + true, + ); + }); + it('rewrites only the reported occurrence, within the supported versions', () => { const text = 'oldName(oldName(1));'; const out = pack.transform(text, finding(1, 9), { diff --git a/packages/core/src/packs/contract.ts b/packages/core/src/packs/contract.ts index 3fc0449..394f0b6 100644 --- a/packages/core/src/packs/contract.ts +++ b/packages/core/src/packs/contract.ts @@ -35,6 +35,12 @@ export interface PackMeta { sources: readonly PackSource[]; /** Who answers for the pack: a GitHub handle (`@octocat`) or `uptide-dev`. */ maintainer: string; + /** + * The real breaking sites `verified` needs, when not `VERIFIED_MIN_BREAKING_SITES`: only for a + * major whose guide lists fewer breaking changes, and only with the reason written under + * `## Breaking-site minimum` in the pack's RESEARCH.md (`pack test` checks it is there). + */ + minBreakingSites?: number; } /** One place a rule or a behavior note applies, found in a file's text. */ @@ -100,6 +106,11 @@ export interface Pack extends MigrationPack { readonly behavior: readonly BehaviorNote[]; /** What the agent is told for every site no rule rewrites; each rule's `guide` comes first. */ readonly instructions: string; + /** + * Rules a hand-written `transform` edits without a `rewrite` of their own (Stripe's pin): + * mechanical for the verified gate, so each needs a real site like any rewrite. + */ + readonly mechanical?: readonly string[]; } /** @@ -225,6 +236,8 @@ export const VERIFIED_MIN_BEHAVIOR_SITES = 5; */ export interface GateEvidence { repos: number; + /** The pack's own minimum (`meta.minBreakingSites`), else `VERIFIED_MIN_BREAKING_SITES`. */ + minBreakingSites?: number; /** Expected sites under a rule or `generic`: what the compiler or a rule sees, not a note. */ ruleSites: number; breakingSites: number; @@ -262,9 +275,10 @@ export function verifiedGate(e: GateEvidence): GateResult { `${e.breaking.falsePositives} false positive${e.breaking.falsePositives === 1 ? '' : 's'} among breaking findings`, ); if (cls === 'breaking') { - if (e.breakingSites < VERIFIED_MIN_BREAKING_SITES) + const min = e.minBreakingSites ?? VERIFIED_MIN_BREAKING_SITES; + if (e.breakingSites < min) reasons.push( - `${sites(e.breakingSites, 'breaking')} in public ground truth, verified needs ${VERIFIED_MIN_BREAKING_SITES}`, + `${sites(e.breakingSites, 'breaking')} in public ground truth, verified needs ${min}`, ); if (e.breaking.truePositives === 0) reasons.push('no breaking site found'); } else { diff --git a/packages/core/src/packs/fixability.ts b/packages/core/src/packs/fixability.ts index fd9ed68..5440fe0 100644 --- a/packages/core/src/packs/fixability.ts +++ b/packages/core/src/packs/fixability.ts @@ -1,5 +1,47 @@ import type { Finding } from '../domain/report.js'; -import type { MigrationPack } from './types.js'; +import { ruleFor } from './contract.js'; +import type { MigrationPack, TransformResult } from './types.js'; + +/** The rule a site, or the edit made there, belongs to. */ +function ruleIdOf(pack: MigrationPack, finding: Finding, result?: TransformResult) { + return result?.rule ?? finding.rule ?? pack.ruleOf?.(finding) ?? ruleFor(pack.rules, finding)?.id; +} + +/** A site of a rule no real upgrade exercised: assisted, whatever its rewrite says (UPT-48). */ +export function isUnexercised( + pack: MigrationPack, + finding: Finding, + result?: TransformResult, +): boolean { + const id = ruleIdOf(pack, finding, result); + return id !== undefined && (pack.unexercised ?? []).includes(id); +} + +/** + * The pack as its record allows it to run: an edit of an unexercised rule is refused, so `fix` + * hands the site to the agent with the guide. Detection is unchanged. + */ +export function withUnexercised( + pack: MigrationPack, + unexercised: readonly string[] | undefined, +): MigrationPack { + if (!unexercised?.length) return pack; + const wrapped: MigrationPack = { + ...pack, + unexercised, + transform(text, finding, context) { + const result = pack.transform(text, finding, context); + if (!result.applied || !isUnexercised(wrapped, finding, result)) return result; + return { + text, + applied: false, + reason: `no real upgrade in the ground truth exercised the ${ruleIdOf(pack, finding, result)} rewrite yet: assisted`, + }; + }, + }; + return wrapped; +} + /** Use the actual transform as a dry run, including its safety guards. */ export function packFixability(finding: Finding, source: string, pack: MigrationPack): Finding { if (!['breaking', 'unverified', 'deprecated'].includes(finding.severity)) return finding; @@ -8,5 +50,9 @@ export function packFixability(finding: Finding, source: string, pack: Migration to: finding.change.to, includeDeprecated: true, }); - return result.applied ? { ...finding, fixability: 'mechanical' } : finding; + if (result.applied) return { ...finding, fixability: 'mechanical' }; + // A site its detection already called mechanical (a rule's own detect) is not, unexercised. + return finding.fixability === 'mechanical' && isUnexercised(pack, finding) + ? { ...finding, fixability: 'assisted' } + : finding; } diff --git a/packages/core/src/packs/index.ts b/packages/core/src/packs/index.ts index ce16f6a..8eb210a 100644 --- a/packages/core/src/packs/index.ts +++ b/packages/core/src/packs/index.ts @@ -1,5 +1,6 @@ import { UptideError } from '../errors.js'; import { type PackStatus, type RegisteredPack, recordedStatus } from './contract.js'; +import { withUnexercised } from './fixability.js'; import { REGISTRY } from './registry.js'; import type { MigrationPack } from './types.js'; @@ -21,7 +22,7 @@ export function activePacks(allowCandidate = false): readonly MigrationPack[] { return REGISTRY.filter( (e) => packStatus(e) === 'verified' || (allowCandidate && e.verification?.status === 'candidate'), - ).map((e) => e.pack); + ).map((e) => withUnexercised(e.pack, e.verification?.unexercised)); } export function activePack(name: string, allowCandidate = false): MigrationPack | undefined { diff --git a/packages/core/src/packs/stripe/index.ts b/packages/core/src/packs/stripe/index.ts index c2baac3..e4c193a 100644 --- a/packages/core/src/packs/stripe/index.ts +++ b/packages/core/src/packs/stripe/index.ts @@ -325,6 +325,8 @@ export const stripePack: Pack = { }, ], instructions: SDK_GUIDE, + // `transform` pins an unpinned client and widens a fixture cast; no rule has a `rewrite`. + mechanical: ['api-version-unpinned', 'fixture-cast'], transform: (text, finding, context) => { // A client without apiVersion says the version the target SDK's types describe: with the // bump that is the only honest pin. Staying on the old version is `--pin-current-api`. diff --git a/packages/core/src/packs/tooling/pack-test.test.ts b/packages/core/src/packs/tooling/pack-test.test.ts index f00035e..33069dc 100644 --- a/packages/core/src/packs/tooling/pack-test.test.ts +++ b/packages/core/src/packs/tooling/pack-test.test.ts @@ -1,4 +1,6 @@ -import { readFileSync } from 'node:fs'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { expect, it } from 'vitest'; import type { CheckReport, Finding } from '../../domain/report.js'; @@ -8,6 +10,7 @@ import { registeredPacks } from '../index.js'; import { companionOutcome, gateOf, + minimumProblems, overclaims, predictedSites, type RepoScore, @@ -202,6 +205,7 @@ it('gates on real sites only: a fixture repository scores, never counts, and nev { id: 'look', summary: '', severity: 'breaking' as const }, ], behavior: [], + meta: {}, } as unknown as Parameters[0]; const site = (file: string, rule: string) => ({ file, @@ -252,3 +256,23 @@ it('gates on real sites only: a fixture repository scores, never counts, and nev const broken = { ...third, error: 'clone failed' }; expect(gateOf(pack, truth, [real(0), real(1), broken]).breakingSites).toBe(4); }); + +it('takes a lower breaking-site minimum only with its reason written in RESEARCH.md', () => { + const dir = mkdtempSync(join(tmpdir(), 'uptide-minimum-')); + const needs = [ + 'meta.minBreakingSites needs its reason under "## Breaking-site minimum" in RESEARCH.md', + ]; + expect(minimumProblems(undefined, dir)).toEqual([]); + expect(minimumProblems(3, dir)).toEqual(needs); + writeFileSync( + join(dir, 'RESEARCH.md'), + '# x\n\n## Breaking-site minimum\n\n## Sources\n\nguide\n', + ); + expect(minimumProblems(3, dir)).toEqual(needs); + writeFileSync( + join(dir, 'RESEARCH.md'), + '# x\n\n## Breaking-site minimum\n\nThe 2.0 guide lists three breaking changes.\n\n## Sources\n', + ); + expect(minimumProblems(3, dir)).toEqual([]); + expect(minimumProblems(0, dir)[0]).toContain('at least 1'); +}); diff --git a/packages/core/src/packs/tooling/pack-test.ts b/packages/core/src/packs/tooling/pack-test.ts index a5f322d..23cd8fb 100644 --- a/packages/core/src/packs/tooling/pack-test.ts +++ b/packages/core/src/packs/tooling/pack-test.ts @@ -399,6 +399,7 @@ export async function testPack( const problems: string[] = []; if (!pack.meta.sources.length) problems.push('meta.sources lists no changelog or guide'); if (!pack.meta.maintainer) problems.push('meta.maintainer is empty'); + problems.push(...minimumProblems(pack.meta.minBreakingSites, dir)); problems.push(...companionProblems(pack.companions, pack.name)); problems.push( ...companionProblems(pack.releaseGroup, pack.name).map((p) => @@ -489,7 +490,7 @@ export async function testPack( * scores, never counts), and so are the true positives that exercise a mechanical rule. */ export function gateOf( - pack: Pick, + pack: Pick, truth: GroundTruth, repos: RepoScore[], ): PackTestReport['gate'] { @@ -502,19 +503,21 @@ export function gateOf( const ruleSites = expected.length - behaviorSites; const all = scoreTruth(pack, scored); const { rules } = scoreTruth(pack, real); + const ids = [...pack.rules.filter((r) => r.rewrite).map((r) => r.id), ...(pack.mechanical ?? [])]; const mechanical = Object.fromEntries( - pack.rules - .filter((r) => r.rewrite) - .map((r) => [ - r.id, - { - truePositives: rules[r.id]?.truePositives ?? 0, - falsePositives: all.rules[r.id]?.falsePositives ?? 0, - }, - ]), + ids.map((id) => [ + id, + { + truePositives: rules[id]?.truePositives ?? 0, + falsePositives: all.rules[id]?.falsePositives ?? 0, + }, + ]), ); const gate = verifiedGate({ repos: publicRepos(truth).length, + ...(pack.meta.minBreakingSites !== undefined + ? { minBreakingSites: pack.meta.minBreakingSites } + : {}), ruleSites, breakingSites, behaviorSites, @@ -525,6 +528,24 @@ export function gateOf( return { ...gate, breakingSites, behaviorSites }; } +/** + * `meta.minBreakingSites`: a whole number of at least one, and its reason written under + * `## Breaking-site minimum` in the pack's RESEARCH.md. + */ +export function minimumProblems(min: number | undefined, dir: string): string[] { + if (min === undefined) return []; + if (!Number.isInteger(min) || min < 1) + return [`meta.minBreakingSites must be a whole number of at least 1, not ${min}`]; + const file = join(dir, 'RESEARCH.md'); + const text = existsSync(file) ? readFileSync(file, 'utf8') : ''; + const start = text.search(/^## Breaking-site minimum[ \t]*$/m); + const body = start < 0 ? '' : text.slice(start).split('\n').slice(1).join('\n'); + const reason = body.split(/^## /m)[0]?.trim(); + return reason + ? [] + : ['meta.minBreakingSites needs its reason under "## Breaking-site minimum" in RESEARCH.md']; +} + /** * Whether a record claims more than the evidence supports: verified where the run measures * candidate. A record may claim less (candidate where the run measures verified): the runtime diff --git a/packages/core/src/packs/types.ts b/packages/core/src/packs/types.ts index 74e4da0..b59a940 100644 --- a/packages/core/src/packs/types.ts +++ b/packages/core/src/packs/types.ts @@ -126,6 +126,11 @@ export interface PackCompanion { export interface MigrationPack { readonly behavior?: readonly BehaviorNote[]; readonly name: string; + /** + * Rules whose edit no real upgrade in the ground truth exercised (`verification.json`): + * their sites are assisted, never rewritten. Set by the registry from the record. + */ + readonly unexercised?: readonly string[]; /** * Packages that always move with this one, even when the installed version of each already * accepts the target. Each carries the `source` that says so: the URL of an official diff --git a/scripts/docs-packs.ts b/scripts/docs-packs.ts index aab8eb6..c51141b 100644 --- a/scripts/docs-packs.ts +++ b/scripts/docs-packs.ts @@ -40,11 +40,11 @@ export function packsTable(): string { pack.meta.from === pack.meta.to ? rangeLabel(pack.meta.from) : `${rangeLabel(pack.meta.from)} → ${rangeLabel(pack.meta.to)}`; - return `| \`${pack.meta.package}\` | ${range} | ${rateLabel(verification.breaking, 'precision')} | ${rateLabel(verification.breaking, 'recall')} | ${repos.join(', ')} | ${verification.status} |`; + return `| \`${pack.meta.package}\` | ${range} | ${rateLabel(verification.breaking, 'precision')} | ${rateLabel(verification.breaking, 'recall')} | ${verification.breakingSites ?? '—'} | ${repos.join(', ')} | ${verification.status} |`; }); return [ - '| Package | Range | Precision | Recall | Ground-truth repositories | Status |', - '| --- | --- | ---: | ---: | --- | --- |', + '| Package | Range | Precision | Recall | Breaking sites | Ground-truth repositories | Status |', + '| --- | --- | ---: | ---: | ---: | --- | --- |', ...rows, ].join('\n'); } From 02f30195a1670f9230b47202acaffda8eed97395 Mon Sep 17 00:00:00 2001 From: Lucas Braum Date: Wed, 14 Oct 2026 23:57:20 -0700 Subject: [PATCH 3/4] feat(packs): record the UPT-48 gate in every pack; Sentry browser and react are candidates pack test --write on every pack but joi and redis (both stay candidate by choice). Each record gains its class, real breaking and behavior-note site counts, and the unexercised mechanical rules, which check and fix now run as assisted. Only two statuses change: @sentry/browser (3 real breaking sites) and @sentry/react (2) become candidates; the Sentry RESEARCH.md says what they need and where to look. No precision, recall or count changed in any record. zod: no merged public 3 -> 4 upgrade that read formErrors/.errors off an untyped catch binding was found, so error-members runs as assisted; no ground truth was added. stripe's pin and fixture-cast edits have no real site and run as assisted too. The README packs table shows each pack's breaking sites. --write formats a record as biome does, so a written record passes lint. The fix machinery tests keep zod's required_error rewrite as written (it is assisted in the registry now). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lucas Braum --- .changeset/stricter-verified-gate.md | 2 +- CONTRIBUTING.md | 9 +++-- README.md | 36 +++++++++---------- docs/packs.md | 13 +++++-- packages/cli/src/readme.test.ts | 2 +- packages/core/src/fix/fix.test.ts | 7 ++++ packages/core/src/fix/isolate.test.ts | 7 ++++ packages/core/src/packs/ai/verification.json | 6 +++- packages/core/src/packs/contract.test.ts | 7 ++++ .../core/src/packs/ioredis/verification.json | 6 +++- .../src/packs/lucide-react/verification.json | 6 +++- .../core/src/packs/next/verification.json | 6 +++- .../src/packs/react-router/verification.json | 6 +++- .../core/src/packs/react/verification.json | 6 +++- .../packs/sentry-browser/verification.json | 8 +++-- .../src/packs/sentry-nextjs/verification.json | 13 ++++++- .../src/packs/sentry-react/verification.json | 14 ++++++-- packages/core/src/packs/sentry/RESEARCH.md | 12 +++++++ .../core/src/packs/stripe/verification.json | 6 +++- .../core/src/packs/tooling/pack-test.test.ts | 8 +++++ packages/core/src/packs/tooling/pack-test.ts | 17 ++++++++- .../core/src/packs/vitest/verification.json | 6 +++- packages/core/src/packs/zod/verification.json | 6 +++- 23 files changed, 168 insertions(+), 41 deletions(-) diff --git a/.changeset/stricter-verified-gate.md b/.changeset/stricter-verified-gate.md index e26f4fd..b7f8854 100644 --- a/.changeset/stricter-verified-gate.md +++ b/.changeset/stricter-verified-gate.md @@ -2,4 +2,4 @@ "uptide": minor --- -`uptide pack test` applies a stricter verified gate: at least five expected breaking sites across the public ground-truth repositories, besides two repositories and no breaking false positive. Upgrades the compiler cannot see are gated on their behavior notes instead. Mechanical rules no real site exercises are listed as unexercised. The output shows the gate's class, site counts and every reason a pack is a candidate. +`uptide pack test` applies a stricter verified gate: at least five expected breaking sites across the public ground-truth repositories, besides two repositories and no breaking false positive. Upgrades the compiler cannot see are gated on their behavior notes instead. Mechanical rules no real site exercises are recorded as unexercised, and `check` and `fix` run their sites as assisted. `@sentry/browser` and `@sentry/react` become candidates (fewer than five real breaking sites). A pack can lower the minimum only with a written reason in its RESEARCH.md. The README packs table shows each pack's breaking-site count. The output shows the gate's class, site counts and every reason a pack is a candidate. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 177eede..8f6ec0b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,7 +98,7 @@ Rules for a pack change: expected breaking sites across them, and no false positive among its breaking findings (docs/packs.md has the behavior-only variant); until then it ships as a candidate, and the CLI treats the dependency as generic. A rewrite no real site exercises is listed - as unexercised: give each mechanical rule at least one real site. + as unexercised and runs as assisted: give each mechanical rule at least one real site. 5. Server API changes (Stripe API versions) are never rewritten by rule. ## Write a pack @@ -125,8 +125,9 @@ open a **Pack request** issue, and say you are writing it. (`// @uptide `, `keep` for one it must leave alone) and `after.ts`, what the rules must produce. `pnpm uptide pack test --fixtures-only` runs them in seconds; `--update-fixtures` writes each `after.ts` from what the rules do, for you to read. -5. **Find ground truth.** Two public repositories that made this upgrade in a commit you - can point at. Pin the commit before it, and draft the expected findings from their own +5. **Find ground truth.** Two or more public repositories that made this upgrade in a + commit you can point at, with at least five breaking sites between them and a real site + for every rule that rewrites. Pin the commit before it, and draft the expected findings from their own compiler and their own upgrade commit: ```sh pnpm packs:truth owner/name --to @@ -146,6 +147,8 @@ open a **Pack request** issue, and say you are writing it. A false negative is a site to cover or a known gap to say in the pull request. A site expected under a behavior note scores at the note's `severity`, and behavior-note sites get their own precision and recall: a missed "needs a look" is not a missed break. + The `gate` line says the class, the real site counts, and why the pack is a candidate if + it is one; rules it lists as unexercised run as assisted until a real site covers them. 7. **Smoke-test locally** without editing `verification.json`: ```sh pnpm uptide check --allow-candidate --cwd /path/to/local-fixture diff --git a/README.md b/README.md index 5f686f4..dd26b9b 100644 --- a/README.md +++ b/README.md @@ -50,26 +50,26 @@ Everything else: [getting started](docs/getting-started.md). A pack is everything Uptide knows about one upgrade: rules, a guide for the agent, behavior checks, and ground truth from public repositories it is scored against. Precision and recall are for breaking findings on that ground truth, as recorded by `uptide pack test`; a pack is -verified with two or more public repositories and no false positive. The contract: -[migration packs](docs/packs.md). +verified with two or more public repositories, five real breaking sites, and no false positive. +The contract: [migration packs](docs/packs.md). -| Package | Range | Precision | Recall | Ground-truth repositories | Status | -| --- | --- | ---: | ---: | --- | --- | -| `ai` | 6.x → 7.x | 100% | 82% | vercel/chatbot, miurla/morphic | verified | -| `ioredis` | 5.x → 6.x | n/a | 0% | ferni-ai/voiceai, syzygyhack/open-foundry, simondietschi/dokunc, safe-global/safe-client-gateway, kazumadev619-dev/FishingConditionsApp | candidate | -| `joi` | 17.x → 18.x | n/a | n/a | edtech4good/edtech-lms-api, senacor/azure-function-middleware | candidate | -| `lucide-react` | 0.x → 1.x | 100% | 100% | tradeblocks-org/tradeblocks, spizeck/SFPCA, cubicsteak/keep | verified | -| `next` | 15.x → 16.x | 100% | 85% | sambecker/exif-photo-blog, stoczec/wortkarte | verified | -| `react` | 18.x → 19.x | 100% | 69% | excalidraw/excalidraw, tldraw/tldraw | verified | -| `react-router` | 6.x → 7.x | 100% | 88% | Arize-ai/phoenix, getlago/lago-front | verified | -| `redis` | >=4 <6 → >=5 <7 | n/a | n/a | deanmoses/tacocat-gallery-sam, swoofer/mcp-coordinator, deanmoses/tacocat-gallery-sam, UNN-Devotek/FCM-Fallout-Chat-Mod, tearingItUp786/blog | candidate | -| `@sentry/browser` | 10.x → 11.x | 100% | 100% | Automancer-Ltd/automancer-site, sbomify/sbomify | verified | -| `@sentry/nextjs` | 10.x → 11.x | 100% | 100% | cohstats/coh3-stats, xdoubleu/tools.xdoubleu.com, maximunited/yomu | verified | -| `@sentry/react` | 10.x → 11.x | 100% | 100% | owine/youth-activity-scheduler, successible/cleanslate | verified | -| `stripe` | 14 and newer | 100% | 90% | unkeyed/unkey, nextjs/saas-starter | verified | -| `vitest` | 4.x → 5.x | 100% | 50% | votingworks/vxsuite, vitorvasc/opentelemetry-ecosystem-explorer | verified | -| `zod` | 3.x → 4.x | 100% | 74% | ugurkocde/AwesomeIntune, howardyang2009/PATH, band-ai/band-sdk-typescript | verified | +| Package | Range | Precision | Recall | Breaking sites | Ground-truth repositories | Status | +| --- | --- | ---: | ---: | ---: | --- | --- | +| `ai` | 6.x → 7.x | 100% | 82% | 11 | vercel/chatbot, miurla/morphic | verified | +| `ioredis` | 5.x → 6.x | n/a | 0% | 2 | ferni-ai/voiceai, syzygyhack/open-foundry, simondietschi/dokunc, safe-global/safe-client-gateway, kazumadev619-dev/FishingConditionsApp | candidate | +| `joi` | 17.x → 18.x | n/a | n/a | — | edtech4good/edtech-lms-api, senacor/azure-function-middleware | candidate | +| `lucide-react` | 0.x → 1.x | 100% | 100% | 11 | tradeblocks-org/tradeblocks, spizeck/SFPCA, cubicsteak/keep | verified | +| `next` | 15.x → 16.x | 100% | 85% | 13 | sambecker/exif-photo-blog, stoczec/wortkarte | verified | +| `react` | 18.x → 19.x | 100% | 69% | 130 | excalidraw/excalidraw, tldraw/tldraw | verified | +| `react-router` | 6.x → 7.x | 100% | 88% | 18 | Arize-ai/phoenix, getlago/lago-front | verified | +| `redis` | >=4 <6 → >=5 <7 | n/a | n/a | — | deanmoses/tacocat-gallery-sam, swoofer/mcp-coordinator, deanmoses/tacocat-gallery-sam, UNN-Devotek/FCM-Fallout-Chat-Mod, tearingItUp786/blog | candidate | +| `@sentry/browser` | 10.x → 11.x | 100% | 100% | 3 | Automancer-Ltd/automancer-site, sbomify/sbomify | candidate | +| `@sentry/nextjs` | 10.x → 11.x | 100% | 100% | 8 | cohstats/coh3-stats, xdoubleu/tools.xdoubleu.com, maximunited/yomu | verified | +| `@sentry/react` | 10.x → 11.x | 100% | 100% | 2 | owine/youth-activity-scheduler, successible/cleanslate | candidate | +| `stripe` | 14 and newer | 100% | 90% | 16 | unkeyed/unkey, nextjs/saas-starter | verified | +| `vitest` | 4.x → 5.x | 100% | 50% | 32 | votingworks/vxsuite, vitorvasc/opentelemetry-ecosystem-explorer | verified | +| `zod` | 3.x → 4.x | 100% | 74% | 85 | ugurkocde/AwesomeIntune, howardyang2009/PATH, band-ai/band-sdk-typescript | verified | Any other dependency gets the same analysis without a pack, and `fix` migrates it with the diff --git a/docs/packs.md b/docs/packs.md index dd46ed7..81b9fb8 100644 --- a/docs/packs.md +++ b/docs/packs.md @@ -308,8 +308,14 @@ pack is a candidate. A rule with a `rewrite` that no public repository's ground truth exercises (no true positive) is **unexercised**: `pack test` lists it and records it in `verification.json` -(`unexercised`), because nothing real confirms its rewrite yet. Its sites are meant to run as -assisted; detection is unaffected, and so is the pack's status. +(`unexercised`), because nothing real confirms its rewrite yet. `check` and `fix` read the record: +its sites are assisted (the agent, with the rule's guide), never rewritten. Detection is +unaffected, and so is the pack's status. A pack whose `transform` edits for a rule that has no +`rewrite` of its own (Stripe's pin) lists that rule in `mechanical`, so the gate counts it too. + +A pack may ask for fewer real breaking sites with `meta.minBreakingSites`, only for a major whose +guide lists fewer breaking changes, and only with the reason written under +`## Breaking-site minimum` in its RESEARCH.md; `pack test` fails a pack that sets it without one. Anything else is a **candidate**: @@ -324,7 +330,8 @@ Anything else is a **candidate**: The CLI reads the status from `verification.json`, which `pack test --write` writes and which records the digest of the ground-truth file it was measured against, with precision, recall and the counts behind them for breaking findings and for behavior-note sites -(`behavior`, absent from records written before it). The gate reads the breaking ones only. CI runs +(`behavior`, absent from records written before it), the gate's class, the real breaking and +behavior-note site counts, and the unexercised rules. CI runs `pack test` for every pack on every pull request that touches packs and fails when the record claims verified where the evidence measures candidate, so the label cannot outlive what supports it. diff --git a/packages/cli/src/readme.test.ts b/packages/cli/src/readme.test.ts index eecb17c..9632679 100644 --- a/packages/cli/src/readme.test.ts +++ b/packages/cli/src/readme.test.ts @@ -73,7 +73,7 @@ describe('README', () => { it('lists the verified packs from a generated table, and privacy in four lines plus the table', () => { const packs = section('', ''); expect(packs).toContain( - '| Package | Range | Precision | Recall | Ground-truth repositories | Status |', + '| Package | Range | Precision | Recall | Breaking sites | Ground-truth repositories | Status |', ); for (const name of ['`zod`', '`stripe`', '`ai`']) expect(packs).toContain(`| ${name} |`); const privacy = section('## Privacy', '## Documentation'); diff --git a/packages/core/src/fix/fix.test.ts b/packages/core/src/fix/fix.test.ts index 9fe15a4..9d2e3a0 100644 --- a/packages/core/src/fix/fix.test.ts +++ b/packages/core/src/fix/fix.test.ts @@ -25,6 +25,13 @@ import { } from './verify.js'; import { bumpCatalog, bumpVersions } from './versions.js'; +// zod's required_error rewrite has no real ground-truth site yet, so the registry runs it as +// assisted (UPT-48). These tests exercise the fix machinery with that rule as written. +vi.mock('../packs/fixability.js', async (importOriginal) => ({ + ...(await importOriginal()), + withUnexercised: (pack: unknown) => pack, +})); + const scratch = mkdtempSync(join(tmpdir(), 'uptide-fix-tests-')); afterAll(() => rmSync(scratch, { recursive: true, force: true })); const fixture = () => zodFixture(scratch); diff --git a/packages/core/src/fix/isolate.test.ts b/packages/core/src/fix/isolate.test.ts index b6c587a..3a1705a 100644 --- a/packages/core/src/fix/isolate.test.ts +++ b/packages/core/src/fix/isolate.test.ts @@ -30,6 +30,13 @@ import { command, git, quietEnv } from './process.js'; import { zodFixture } from './test-fixture.js'; import type { FixRequest } from './types.js'; +// zod's required_error rewrite has no real ground-truth site yet, so the registry runs it as +// assisted (UPT-48). These tests exercise the fix machinery with that rule as written. +vi.mock('../packs/fixability.js', async (importOriginal) => ({ + ...(await importOriginal()), + withUnexercised: (pack: unknown) => pack, +})); + const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'uptide-isolate-'))); // Clones made by these tests live under the scratch directory, not in the machine's temporary root. process.env.UPTIDE_RUNS_DIR = join(scratch, 'runs'); diff --git a/packages/core/src/packs/ai/verification.json b/packages/core/src/packs/ai/verification.json index 32312cd..57fc08b 100644 --- a/packages/core/src/packs/ai/verification.json +++ b/packages/core/src/packs/ai/verification.json @@ -15,5 +15,9 @@ "falsePositives": 0, "truePositives": 0, "falseNegatives": 0 - } + }, + "class": "breaking", + "breakingSites": 11, + "behaviorSites": 0, + "unexercised": [] } diff --git a/packages/core/src/packs/contract.test.ts b/packages/core/src/packs/contract.test.ts index df171a2..84fb39e 100644 --- a/packages/core/src/packs/contract.test.ts +++ b/packages/core/src/packs/contract.test.ts @@ -86,6 +86,13 @@ describe('every registered pack meets the contract', () => { .map((e) => e.pack.name); expect(activePacks().map((p) => p.name)).toEqual(verified); }); + + it('runs each active pack with the unexercised rules its record lists', () => { + for (const pack of activePacks()) { + const entry = registeredPacks().find((e) => e.pack.name === pack.name); + expect(pack.unexercised ?? [], pack.name).toEqual(entry?.verification.unexercised ?? []); + } + }); }); describe('the verified gate', () => { diff --git a/packages/core/src/packs/ioredis/verification.json b/packages/core/src/packs/ioredis/verification.json index 7e1a7bb..3ba3d56 100644 --- a/packages/core/src/packs/ioredis/verification.json +++ b/packages/core/src/packs/ioredis/verification.json @@ -15,5 +15,9 @@ "falsePositives": 3, "truePositives": 7, "falseNegatives": 6 - } + }, + "class": "behavior", + "breakingSites": 2, + "behaviorSites": 13, + "unexercised": [] } diff --git a/packages/core/src/packs/lucide-react/verification.json b/packages/core/src/packs/lucide-react/verification.json index 12a9308..8cc0aa8 100644 --- a/packages/core/src/packs/lucide-react/verification.json +++ b/packages/core/src/packs/lucide-react/verification.json @@ -15,5 +15,9 @@ "falsePositives": 0, "truePositives": 0, "falseNegatives": 0 - } + }, + "class": "breaking", + "breakingSites": 11, + "behaviorSites": 0, + "unexercised": ["renamed-icons"] } diff --git a/packages/core/src/packs/next/verification.json b/packages/core/src/packs/next/verification.json index abffd1c..c952f3b 100644 --- a/packages/core/src/packs/next/verification.json +++ b/packages/core/src/packs/next/verification.json @@ -15,5 +15,9 @@ "falsePositives": 0, "truePositives": 0, "falseNegatives": 0 - } + }, + "class": "breaking", + "breakingSites": 13, + "behaviorSites": 0, + "unexercised": ["stable-cache-exports"] } diff --git a/packages/core/src/packs/react-router/verification.json b/packages/core/src/packs/react-router/verification.json index f10f776..af8447a 100644 --- a/packages/core/src/packs/react-router/verification.json +++ b/packages/core/src/packs/react-router/verification.json @@ -15,5 +15,9 @@ "falsePositives": 0, "truePositives": 0, "falseNegatives": 0 - } + }, + "class": "breaking", + "breakingSites": 18, + "behaviorSites": 0, + "unexercised": [] } diff --git a/packages/core/src/packs/react/verification.json b/packages/core/src/packs/react/verification.json index 0842e0c..9ae480b 100644 --- a/packages/core/src/packs/react/verification.json +++ b/packages/core/src/packs/react/verification.json @@ -15,5 +15,9 @@ "falsePositives": 0, "truePositives": 0, "falseNegatives": 0 - } + }, + "class": "breaking", + "breakingSites": 130, + "behaviorSites": 0, + "unexercised": [] } diff --git a/packages/core/src/packs/sentry-browser/verification.json b/packages/core/src/packs/sentry-browser/verification.json index c7b547d..99c2aad 100644 --- a/packages/core/src/packs/sentry-browser/verification.json +++ b/packages/core/src/packs/sentry-browser/verification.json @@ -1,5 +1,5 @@ { - "status": "verified", + "status": "candidate", "truth": "sha256-fde34dee257f30c945bdf9419815bf7f152f05c66892011da0fb4dffa3857dc0", "repos": 2, "breaking": { @@ -15,5 +15,9 @@ "falsePositives": 0, "truePositives": 0, "falseNegatives": 0 - } + }, + "class": "breaking", + "breakingSites": 3, + "behaviorSites": 0, + "unexercised": ["enable-metrics", "integration-names", "renamed-exports", "streamed-span-wrapper"] } diff --git a/packages/core/src/packs/sentry-nextjs/verification.json b/packages/core/src/packs/sentry-nextjs/verification.json index 5d3e3c6..98f08b2 100644 --- a/packages/core/src/packs/sentry-nextjs/verification.json +++ b/packages/core/src/packs/sentry-nextjs/verification.json @@ -15,5 +15,16 @@ "falsePositives": 0, "truePositives": 0, "falseNegatives": 0 - } + }, + "class": "breaking", + "breakingSites": 8, + "behaviorSites": 0, + "unexercised": [ + "enable-logs", + "enable-metrics", + "http-integration-options", + "integration-names", + "renamed-exports", + "streamed-span-wrapper" + ] } diff --git a/packages/core/src/packs/sentry-react/verification.json b/packages/core/src/packs/sentry-react/verification.json index 5994af5..d480312 100644 --- a/packages/core/src/packs/sentry-react/verification.json +++ b/packages/core/src/packs/sentry-react/verification.json @@ -1,5 +1,5 @@ { - "status": "verified", + "status": "candidate", "truth": "sha256-de564278fba5bea1bf2ff31af11745f79e4bcfa063889715bcaa23e3d0ff9b13", "repos": 2, "breaking": { @@ -15,5 +15,15 @@ "falsePositives": 0, "truePositives": 0, "falseNegatives": 0 - } + }, + "class": "breaking", + "breakingSites": 2, + "behaviorSites": 0, + "unexercised": [ + "enable-logs", + "enable-metrics", + "integration-names", + "renamed-exports", + "streamed-span-wrapper" + ] } diff --git a/packages/core/src/packs/sentry/RESEARCH.md b/packages/core/src/packs/sentry/RESEARCH.md index 278b7ac..48f0aa5 100644 --- a/packages/core/src/packs/sentry/RESEARCH.md +++ b/packages/core/src/packs/sentry/RESEARCH.md @@ -175,6 +175,18 @@ Misses and why: with no compiler signal and no general syntactic rule (renaming the active span is legitimate elsewhere); covered only by the `span-streaming`/`also-review` decisions. +## Candidates under the UPT-48 gate + +`verified` now needs at least five expected breaking sites across the public repositories +(`VERIFIED_MIN_BREAKING_SITES`). `@sentry/browser` has 3 and `@sentry/react` has 2, so both are +recorded `candidate` until their ground truth has five or more real breaking sites each (precision +and recall above are unchanged). `@sentry/nextjs` (8) stays verified. Public repositories to search +later for real 10 → 11 upgrades of these leads: cal.com, midday, rallly, inbox-zero, unkey, twenty. + +The shared rules no real site exercises (`enable-logs`, `enable-metrics`, +`http-integration-options`, `integration-names`, `renamed-exports`, `streamed-span-wrapper`, per lead +as `verification.json` lists them under `unexercised`) run as assisted in `fix`. + ## What the engine could not express 1. **Same-scope packages versioned on their own.** `check/companions.ts` and `list/groups.ts` diff --git a/packages/core/src/packs/stripe/verification.json b/packages/core/src/packs/stripe/verification.json index 6774abc..9d234cd 100644 --- a/packages/core/src/packs/stripe/verification.json +++ b/packages/core/src/packs/stripe/verification.json @@ -15,5 +15,9 @@ "falsePositives": 0, "truePositives": 0, "falseNegatives": 0 - } + }, + "class": "breaking", + "breakingSites": 16, + "behaviorSites": 0, + "unexercised": ["api-version-unpinned", "fixture-cast"] } diff --git a/packages/core/src/packs/tooling/pack-test.test.ts b/packages/core/src/packs/tooling/pack-test.test.ts index 33069dc..fdadf25 100644 --- a/packages/core/src/packs/tooling/pack-test.test.ts +++ b/packages/core/src/packs/tooling/pack-test.test.ts @@ -9,6 +9,7 @@ import type { PackVerification } from '../contract.js'; import { registeredPacks } from '../index.js'; import { companionOutcome, + formatRecord, gateOf, minimumProblems, overclaims, @@ -276,3 +277,10 @@ it('takes a lower breaking-site minimum only with its reason written in RESEARCH expect(minimumProblems(3, dir)).toEqual([]); expect(minimumProblems(0, dir)[0]).toContain('at least 1'); }); + +it('writes a record as the formatter would: a short list inline, a long one a line per item', () => { + for (const entry of registeredPacks()) { + const file = join(root, 'packages/core/src/packs', entry.dir, 'verification.json'); + expect(`${formatRecord(entry.verification)}\n`, entry.dir).toBe(readFileSync(file, 'utf8')); + } +}); diff --git a/packages/core/src/packs/tooling/pack-test.ts b/packages/core/src/packs/tooling/pack-test.ts index 23cd8fb..d5af261 100644 --- a/packages/core/src/packs/tooling/pack-test.ts +++ b/packages/core/src/packs/tooling/pack-test.ts @@ -558,10 +558,25 @@ export function overclaims(recorded: PackStatus, measured: PackStatus): boolean /** `--write`: the record the runtime gate reads, as this run measured it. */ export function writeVerification(root: string, dir: string, record: PackVerification): string { const file = join(packsDir(root), dir, 'verification.json'); - writeFileSync(file, `${JSON.stringify(record, null, 2)}\n`); + writeFileSync(file, `${formatRecord(record)}\n`); return file; } +/** JSON as the repository's formatter writes it: a list of strings on one line when it fits. */ +export function formatRecord(record: PackVerification): string { + return JSON.stringify(record, null, 2).replace( + /^( *"[^"\n]+": )\[\n((?: *"[^"\n]*",?\n)+) *\]/gm, + (whole, head: string, items: string) => { + const line = `${head}[${items + .trim() + .split('\n') + .map((item) => item.trim().replace(/,$/, '')) + .join(', ')}]`; + return line.length <= 100 ? line : whole; + }, + ); +} + /** * A pack whose test never finished (it ran past its deadline, or its process died): failed, * with the reason as its problem, so the run names the pack instead of stalling or vanishing. diff --git a/packages/core/src/packs/vitest/verification.json b/packages/core/src/packs/vitest/verification.json index 71f0b3c..9c27d58 100644 --- a/packages/core/src/packs/vitest/verification.json +++ b/packages/core/src/packs/vitest/verification.json @@ -15,5 +15,9 @@ "falsePositives": 0, "truePositives": 0, "falseNegatives": 13 - } + }, + "class": "breaking", + "breakingSites": 32, + "behaviorSites": 13, + "unexercised": ["removed-entrypoints"] } diff --git a/packages/core/src/packs/zod/verification.json b/packages/core/src/packs/zod/verification.json index 6eb5ce7..357027a 100644 --- a/packages/core/src/packs/zod/verification.json +++ b/packages/core/src/packs/zod/verification.json @@ -15,5 +15,9 @@ "falsePositives": 0, "truePositives": 0, "falseNegatives": 0 - } + }, + "class": "breaking", + "breakingSites": 85, + "behaviorSites": 0, + "unexercised": ["error-members", "error-params"] } From ac063b8df854823ffadcb9c0a0a1ff5306c0e27e Mon Sep 17 00:00:00 2001 From: Lucas Braum Date: Thu, 15 Oct 2026 00:38:48 -0700 Subject: [PATCH 4/4] test(smoke): migrate zod's record-key in the no-LLM smoke; scaffold writes the full record The smoke ran fix without the agent and expected zod's required_error sites to migrate by rule; UPT-48 runs that rewrite as assisted, so the fixtures now use z.record(value) (record-key: mechanical, exercised by real ground truth) with the same site counts, and the smoke checks for that rule's line and the two-argument form. The fix machinery is tested the same way; the gate is unchanged. pack new writes verification.json with the gate's fields (class, site counts, the example rewrite as unexercised) in the formatter's layout, so a fresh pack's first pack test finds its record current. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lucas Braum --- .../fixtures/npm-workspaces/packages/api/src/signup.ts | 6 ++++-- .../npm-workspaces/packages/shared/src/index.ts | 8 +++----- packages/cli/smoke/fixtures/npm/src/schema.ts | 8 +++----- .../cli/smoke/fixtures/pnpm/packages/api/src/signup.ts | 6 ++++-- .../smoke/fixtures/pnpm/packages/shared/src/index.ts | 8 +++----- packages/cli/smoke/fixtures/yarn-berry/src/schema.ts | 8 +++----- packages/cli/smoke/fixtures/yarn/src/schema.ts | 8 +++----- packages/cli/smoke/in-container.mjs | 10 ++++++++-- packages/core/src/packs/tooling/scaffold.ts | 9 +++++++-- 9 files changed, 38 insertions(+), 33 deletions(-) diff --git a/packages/cli/smoke/fixtures/npm-workspaces/packages/api/src/signup.ts b/packages/cli/smoke/fixtures/npm-workspaces/packages/api/src/signup.ts index 360d931..b25f10c 100644 --- a/packages/cli/smoke/fixtures/npm-workspaces/packages/api/src/signup.ts +++ b/packages/cli/smoke/fixtures/npm-workspaces/packages/api/src/signup.ts @@ -1,8 +1,10 @@ import { z } from 'zod'; export const signup = z.object({ - email: z.string({ required_error: 'Email is required' }), - plan: z.enum(['free', 'pro'], { invalid_type_error: 'Unknown plan' }), + email: z.string().min(3), + plan: z.enum(['free', 'pro']), + metadata: z.record(z.string()), + limits: z.record(z.number()), }); export type Signup = z.infer; diff --git a/packages/cli/smoke/fixtures/npm-workspaces/packages/shared/src/index.ts b/packages/cli/smoke/fixtures/npm-workspaces/packages/shared/src/index.ts index e3233ef..a3aff19 100644 --- a/packages/cli/smoke/fixtures/npm-workspaces/packages/shared/src/index.ts +++ b/packages/cli/smoke/fixtures/npm-workspaces/packages/shared/src/index.ts @@ -1,11 +1,9 @@ import { z } from 'zod'; export const user = z.object({ - name: z.string({ - required_error: 'Name is required', - invalid_type_error: 'Name must be a string', - }), - age: z.number({ invalid_type_error: 'Age must be a number' }).int(), + name: z.string(), + labels: z.record(z.string()), + scores: z.record(z.number().int()), }); export type User = z.infer; diff --git a/packages/cli/smoke/fixtures/npm/src/schema.ts b/packages/cli/smoke/fixtures/npm/src/schema.ts index e3233ef..a3aff19 100644 --- a/packages/cli/smoke/fixtures/npm/src/schema.ts +++ b/packages/cli/smoke/fixtures/npm/src/schema.ts @@ -1,11 +1,9 @@ import { z } from 'zod'; export const user = z.object({ - name: z.string({ - required_error: 'Name is required', - invalid_type_error: 'Name must be a string', - }), - age: z.number({ invalid_type_error: 'Age must be a number' }).int(), + name: z.string(), + labels: z.record(z.string()), + scores: z.record(z.number().int()), }); export type User = z.infer; diff --git a/packages/cli/smoke/fixtures/pnpm/packages/api/src/signup.ts b/packages/cli/smoke/fixtures/pnpm/packages/api/src/signup.ts index 360d931..b25f10c 100644 --- a/packages/cli/smoke/fixtures/pnpm/packages/api/src/signup.ts +++ b/packages/cli/smoke/fixtures/pnpm/packages/api/src/signup.ts @@ -1,8 +1,10 @@ import { z } from 'zod'; export const signup = z.object({ - email: z.string({ required_error: 'Email is required' }), - plan: z.enum(['free', 'pro'], { invalid_type_error: 'Unknown plan' }), + email: z.string().min(3), + plan: z.enum(['free', 'pro']), + metadata: z.record(z.string()), + limits: z.record(z.number()), }); export type Signup = z.infer; diff --git a/packages/cli/smoke/fixtures/pnpm/packages/shared/src/index.ts b/packages/cli/smoke/fixtures/pnpm/packages/shared/src/index.ts index e3233ef..a3aff19 100644 --- a/packages/cli/smoke/fixtures/pnpm/packages/shared/src/index.ts +++ b/packages/cli/smoke/fixtures/pnpm/packages/shared/src/index.ts @@ -1,11 +1,9 @@ import { z } from 'zod'; export const user = z.object({ - name: z.string({ - required_error: 'Name is required', - invalid_type_error: 'Name must be a string', - }), - age: z.number({ invalid_type_error: 'Age must be a number' }).int(), + name: z.string(), + labels: z.record(z.string()), + scores: z.record(z.number().int()), }); export type User = z.infer; diff --git a/packages/cli/smoke/fixtures/yarn-berry/src/schema.ts b/packages/cli/smoke/fixtures/yarn-berry/src/schema.ts index e3233ef..a3aff19 100644 --- a/packages/cli/smoke/fixtures/yarn-berry/src/schema.ts +++ b/packages/cli/smoke/fixtures/yarn-berry/src/schema.ts @@ -1,11 +1,9 @@ import { z } from 'zod'; export const user = z.object({ - name: z.string({ - required_error: 'Name is required', - invalid_type_error: 'Name must be a string', - }), - age: z.number({ invalid_type_error: 'Age must be a number' }).int(), + name: z.string(), + labels: z.record(z.string()), + scores: z.record(z.number().int()), }); export type User = z.infer; diff --git a/packages/cli/smoke/fixtures/yarn/src/schema.ts b/packages/cli/smoke/fixtures/yarn/src/schema.ts index e3233ef..a3aff19 100644 --- a/packages/cli/smoke/fixtures/yarn/src/schema.ts +++ b/packages/cli/smoke/fixtures/yarn/src/schema.ts @@ -1,11 +1,9 @@ import { z } from 'zod'; export const user = z.object({ - name: z.string({ - required_error: 'Name is required', - invalid_type_error: 'Name must be a string', - }), - age: z.number({ invalid_type_error: 'Age must be a number' }).int(), + name: z.string(), + labels: z.record(z.string()), + scores: z.record(z.number().int()), }); export type User = z.infer; diff --git a/packages/cli/smoke/in-container.mjs b/packages/cli/smoke/in-container.mjs index 7b44007..c9b65c6 100644 --- a/packages/cli/smoke/in-container.mjs +++ b/packages/cli/smoke/in-container.mjs @@ -149,7 +149,13 @@ for (const fixture of ['npm', 'npm-workspaces', 'pnpm', 'yarn', 'yarn-berry']) { check.out, /zod\s+3\.23\.8 → 4\.6\.5\s+major · (?:--target|latest on npm)\s+verified\s+✗ \d+ breaking/, ); - has(`${manager} check`, check.out, 'New error API (required_error → error)'); + // zod's record-key rewrite: mechanical, exercised by real ground truth (unlike required_error, + // which UPT-48 runs as assisted until a real site covers it). + has( + `${manager} check`, + check.out, + 'z.record(valueSchema) requires an explicit string key schema', + ); matches( `${manager} check`, check.out, @@ -308,7 +314,7 @@ for (const fixture of ['npm', 'npm-workspaces', 'pnpm', 'yarn', 'yarn-berry']) { for (const file of files) expect( `${fixture}: ${file} migrated`, - !/required_error|invalid_type_error/.test(onBranch(repo, zodBranch, file)), + !/z\.record\((?!z\.string\(\), )/.test(onBranch(repo, zodBranch, file)), ); if (fixture === 'npm-workspaces') { expect( diff --git a/packages/core/src/packs/tooling/scaffold.ts b/packages/core/src/packs/tooling/scaffold.ts index aad2ffa..b6520aa 100644 --- a/packages/core/src/packs/tooling/scaffold.ts +++ b/packages/core/src/packs/tooling/scaffold.ts @@ -9,7 +9,7 @@ import { truthDigest, verificationTally, } from '../contract.js'; -import { packsDir } from './pack-test.js'; +import { formatRecord, packsDir } from './pack-test.js'; import { packConstant, packDir, parseRegistry, renderRegistry } from './registry-file.js'; export interface ScaffoldOptions { @@ -166,9 +166,14 @@ export function scaffoldPack(options: ScaffoldOptions): ScaffoldResult { repos: 0, breaking: verificationTally(nothing), behavior: verificationTally(nothing), + class: 'breaking', + breakingSites: 0, + behaviorSites: 0, + // No real site exercises the example rewrite yet. + unexercised: [RULE], }; const verificationFile = join(home, 'verification.json'); - writeFileSync(verificationFile, `${JSON.stringify(verification, null, 2)}\n`); + writeFileSync(verificationFile, `${formatRecord(verification)}\n`); written.splice(written.length - 1, 0, verificationFile); return { dir,