From b9671fe5f56e0a8defc9fa2a6eac84b4979a9920 Mon Sep 17 00:00:00 2001 From: Lucas Braum Date: Thu, 15 Oct 2026 18:48:23 -0700 Subject: [PATCH] fix(packs): pack test counts truth sites listed as folded evidence as covered A ground-truth site that an anchored finding lists as evidence (TypeScript 6 `types` errors folded into changed-defaults, #136's option cascades) now scores as covered, at the site's own rule and severity. Evidence the truth does not name stays unscored. Ground truth unchanged. Re-scored typescript: breaking recall 17% -> 99% (1,379 of 1,399), 0 FP. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lucas Braum --- .changeset/score-folded-evidence.md | 5 +++++ README.md | 2 +- docs/packs.md | 4 +++- .../core/src/packs/tooling/pack-test.test.ts | 9 +++++++++ packages/core/src/packs/tooling/pack-test.ts | 18 ++++++++++++++++-- packages/core/src/packs/typescript/RESEARCH.md | 9 ++++++++- .../src/packs/typescript/verification.json | 6 +++--- 7 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 .changeset/score-folded-evidence.md diff --git a/.changeset/score-folded-evidence.md b/.changeset/score-folded-evidence.md new file mode 100644 index 0000000..92411ba --- /dev/null +++ b/.changeset/score-folded-evidence.md @@ -0,0 +1,5 @@ +--- +"uptide": patch +--- + +`uptide pack test` counts a ground-truth site that a folded finding lists as its evidence as covered by that finding, so recall reflects what `check` reports. The typescript pack's breaking recall goes from 17% to 99% on unchanged ground truth: its TypeScript 6 `types` default errors fold into one tsconfig finding each. diff --git a/README.md b/README.md index 6501701..4234638 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ The contract: [migration packs](docs/packs.md). | `@sentry/nextjs` | 10.x → 11.x | 100% | 100% | 8 | cohstats/coh3-stats, xdoubleu/tools.xdoubleu.com, maximunited/yomu | verified | | `@sentry/react` | 10.x → 11.x | 100% | 100% | 2 | owine/youth-activity-scheduler, successible/cleanslate | candidate | | `stripe` | 14 and newer | 100% | 92% | 21 | unkeyed/unkey, nextjs/saas-starter, starl3xx/wallet-to-social, AtlasDevHQ/atlas | verified | -| `typescript` | 5.x → 6.x | 100% | 17% | 1399 | m-lyon/recipe, robrechtme/recipes, JosephMaynard/dependency-radar, CroffZ/github-release-notifier, SignalK/signalk-derived-data, shogo82148/typescript-action, Borewit/music-metadata | verified | +| `typescript` | 5.x → 6.x | 100% | 99% | 1399 | m-lyon/recipe, robrechtme/recipes, JosephMaynard/dependency-radar, CroffZ/github-release-notifier, SignalK/signalk-derived-data, shogo82148/typescript-action, Borewit/music-metadata | verified | | `vitest` | 4.x → 5.x | 100% | 50% | 32 | votingworks/vxsuite, vitorvasc/opentelemetry-ecosystem-explorer | verified | | `zod` | 3.x → 4.x | 100% | 64% | 219 | ugurkocde/AwesomeIntune, howardyang2009/PATH, band-ai/band-sdk-typescript, vzdc-artcc/website, CANShift/canshift-core | verified | diff --git a/docs/packs.md b/docs/packs.md index 50d1f88..7299d56 100644 --- a/docs/packs.md +++ b/docs/packs.md @@ -140,7 +140,9 @@ site is reported is part of the contract (`reported`): findings are claimed and folds compiler errors that one edit elsewhere resolves into a finding at that edit, as their evidence. The typescript pack folds the "Cannot find name" errors of the 6.0 `types: []` default into one at the tsconfig that leaves - `types` unset. + `types` unset. `pack test` counts a ground-truth site listed as such evidence as + covered, at the site's own rule and severity; evidence the truth does not name is not + scored. - `decision`: the pull request lists it under "Decisions for you", with the files and lines. - `test-follow-up`: an assertion is updated only after the migrated code made that test fail, and the test then has to pass (zod's reworded default messages). diff --git a/packages/core/src/packs/tooling/pack-test.test.ts b/packages/core/src/packs/tooling/pack-test.test.ts index 14840b7..b250af5 100644 --- a/packages/core/src/packs/tooling/pack-test.test.ts +++ b/packages/core/src/packs/tooling/pack-test.test.ts @@ -142,6 +142,15 @@ it('scores an anchored finding at its anchor; the call sites listed as evidence expect(predictedSites(report, aiPack).map((s) => `${s.file}:${s.line} ${s.rule}`)).toEqual([ 'packages/lib/hook.ts:6 tool-context', ]); + // Evidence the ground truth names is covered by the anchor, at the truth's rule and severity; + // evidence it does not name (x.tsx:11) stays unscored. + const expected = [ + { file: 'apps/b/src/y.tsx', line: 59, rule: 'generic', severity: 'breaking' as const }, + ]; + expect(predictedSites(report, aiPack, expected)).toEqual([ + { file: 'apps/b/src/y.tsx', line: 59, rule: 'generic', severity: 'breaking' }, + { file: 'packages/lib/hook.ts', line: 6, rule: 'tool-context', severity: 'breaking' }, + ]); }); it('counts a required move as moved, and a move check reports only a new major makes as explained', () => { diff --git a/packages/core/src/packs/tooling/pack-test.ts b/packages/core/src/packs/tooling/pack-test.ts index df7d7d2..2b4992d 100644 --- a/packages/core/src/packs/tooling/pack-test.ts +++ b/packages/core/src/packs/tooling/pack-test.ts @@ -141,8 +141,14 @@ const key = (s: { file: string; line: number }): string => `${s.file}:${s.line}` /** * The sites `check` shows for the package, from its plan: the findings worth acting on, * grouped under the rule `fix` would apply. Repository-relative, as ground truth is written. + * An `expected` site an anchored finding lists as evidence is shown too (see below). */ -export function predictedSites(report: CheckReport, pack: Pack): ScoredSite[] { +export function predictedSites( + report: CheckReport, + pack: Pack, + expected: ScoredSite[] = [], +): ScoredSite[] { + const wanted = new Map(expected.map((e) => [key(e), e])); const own = new Set([...pack.rules.map((r) => r.id), ...pack.behavior.map((b) => b.id)]); const sites = new Map(); for (const p of report.packages) { @@ -184,6 +190,14 @@ export function predictedSites(report: CheckReport, pack: Pack): ScoredSite[] { // One site, one rule: the more severe group wins, as the plan lists it first. if (!sites.has(key(site))) sites.set(key(site), site); } + // Evidence the ground truth names is covered: `check` reports the line under its anchor + // (TypeScript 6's `types` errors under one tsconfig finding, #136's option cascades), so + // it scores as the site the truth expects, at that site's rule and severity. Evidence the + // truth does not name stays unscored. + for (const k of evidence) { + const site = wanted.get(k); + if (site && !sites.has(k)) sites.set(k, { ...site }); + } } return [...sites.values()].sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line); } @@ -350,7 +364,7 @@ async function scoreRepo( if (installed.length > 0 && !installed.includes(entry.from)) base.installedMismatch = `the lockfile has ${installed.join(', ')}, the ground truth says ${entry.from}`; const own = report.packages.filter((p) => p.name === pack.name); - const predicted = predictedSites(report, pack); + const predicted = predictedSites(report, pack, expected); return { ...base, ...companionOutcome(own, entry.with, (name) => dependsOn(project, dir, name, pack.name)), diff --git a/packages/core/src/packs/typescript/RESEARCH.md b/packages/core/src/packs/typescript/RESEARCH.md index 1666413..b176909 100644 --- a/packages/core/src/packs/typescript/RESEARCH.md +++ b/packages/core/src/packs/typescript/RESEARCH.md @@ -217,6 +217,13 @@ The recall figure measures the original truth's convention (every error line a s the engine's (one site per edit). It is not a count of breaks `check` misses: every folded error is listed under its tsconfig finding. +**Scoring the folded evidence.** Since then `pack test` counts a ground-truth site that an +anchored finding lists as evidence as covered by that finding, at the site's own rule and +severity (`predictedSites`, as for #136's option cascades). The ground truth did not change. +Re-scored 2026-10-15: breaking recall **99% (1,379 of 1,399)**, 0 breaking false positives, +the same as before the fold. The 20 breaking misses are the second-round and same-change lines +listed under "Measured `pack test`". + ## Verified under UPT-48 `pnpm uptide pack test typescript --write` on main (`530f90c`, UPT-48's gate), with its own @@ -226,7 +233,7 @@ is unchanged (same digest as the candidate record). - Repositories: 7 public (gate: ≥2). - Real breaking sites: 1,399 expected (gate: ≥5). - Breaking false positives: 0 (gate: 0). Breaking precision 100%, recall 17% (240 of 1,399; the - folded `types` errors above). + folded `types` errors above), 99% (1,379) once the scorer counts folded evidence (above). - Behavior notes: precision 20%, recall 50% (2 TP, 8 FP, 2 FN), all `changed-defaults` anchors at `tsconfig*.json:1`. Notes do not count against a breaking-class gate. - Per-rule real-world true positives: `base-url` 4, `module-resolution-node10` 2, `target-es5` 1, diff --git a/packages/core/src/packs/typescript/verification.json b/packages/core/src/packs/typescript/verification.json index ecb2dc1..6382c7a 100644 --- a/packages/core/src/packs/typescript/verification.json +++ b/packages/core/src/packs/typescript/verification.json @@ -4,10 +4,10 @@ "repos": 7, "breaking": { "precision": 1, - "recall": 0.1716, + "recall": 0.9857, "falsePositives": 0, - "truePositives": 240, - "falseNegatives": 1159 + "truePositives": 1379, + "falseNegatives": 20 }, "behavior": { "precision": 0.2,