feat(capability-matrix): strict cross-SDK parity score + coverage scope

scripts/capability-matrix/src/generate-site.ts

@@ -90,9 +90,8 @@ function renderArea(loaded: LoadedArea, compliance: Partial<Record<Language, Com
 
   let rows = "";
   for (const [group, features] of groups) {
-    rows += `<tr class="group-row"><td colspan="${LANGUAGES.length + 2}">${esc(group)}</td></tr>\n`;
+    rows += `<tr class="group-row"><td colspan="${LANGUAGES.length + 1}">${esc(group)}</td></tr>\n`;
     for (const f of features) {
-      const fp = parity.perFeature[f.id] ?? 0;
       const nameHtml = specs.has(f.id)
         ? `<a class="feature-spec-link" href="${esc(`${SPEC_GITHUB_BASE}/${f.id.replaceAll(".", "/")}.md`)}" target="_blank" rel="noopener noreferrer">${esc(f.name)}</a>`
         : esc(f.name);
@@ -102,7 +101,6 @@ function renderArea(loaded: LoadedArea, compliance: Partial<Record<Language, Com
           <div class="feature-desc">${esc(f.description)}</div>
         </td>
         ${LANGUAGES.map((l) => statusCell(f, l, compliance)).join("")}
-        <td class="parity-cell ${parityClass(fp)}">${pct(fp)}</td>
       </tr>\n`;
     }
   }
@@ -119,7 +117,6 @@ function renderArea(loaded: LoadedArea, compliance: Partial<Record<Language, Com
           <tr>
             <th class="th-feature">Feature</th>
             ${LANGUAGES.map((l) => `<th class="th-sdk">${esc(LANG_LABELS[l])}</th>`).join("")}
-            <th class="th-parity">Parity</th>
           </tr>
         </thead>
         <tbody>
@@ -220,17 +217,23 @@ export function renderHtml(
     }
     .build-info a { color: #3ECF8E; text-decoration: none; }
     .build-info a:hover { text-decoration: underline; }
-    .overall-badge {
+    .metrics-row {
       display: flex;
-      align-items: center;
-      gap: 0.5rem;
+      flex-wrap: wrap;
+      gap: 0.75rem;
+      margin-bottom: 1.5rem;
+    }
+    .metric-card {
       background: #2d2d2d;
-      padding: 0.5rem 1rem;
+      padding: 0.65rem 1rem;
       border-radius: 8px;
-      font-size: 0.85rem;
+      min-width: 220px;
+      max-width: 320px;
     }
-    .overall-badge .label { color: #888; }
-    .overall-badge .value { font-weight: 700; color: #3ECF8E; font-size: 1.1rem; }
+    .metric-card .metric-top { display: flex; align-items: baseline; gap: 0.5rem; }
+    .metric-card .label { color: #888; font-size: 0.85rem; }
+    .metric-card .value { font-weight: 700; color: #3ECF8E; font-size: 1.1rem; }
+    .metric-card .metric-desc { font-size: 0.72rem; color: #999; margin-top: 0.25rem; line-height: 1.4; }
 
     /* ── SDK score cards ───────────────────────────────────── */
     .sdk-grid {
@@ -352,7 +355,6 @@ export function renderHtml(
     }
     .th-feature { min-width: 200px; }
     .th-sdk { width: 80px; text-align: center; }
-    .th-parity { width: 60px; text-align: center; }
 
     tbody tr:hover { background: #f8fafc; }
 
@@ -416,17 +418,6 @@ export function renderHtml(
       white-space: nowrap;
     }
 
-    /* ── Parity cell ───────────────────────────────────────── */
-    .parity-cell {
-      text-align: center;
-      font-weight: 600;
-      font-size: 0.75rem;
-      font-variant-numeric: tabular-nums;
-    }
-    .parity-high { color: #16a34a; }
-    .parity-mid  { color: #d97706; }
-    .parity-low  { color: #dc2626; }
-
     /* ── SDK score parity colours ──────────────────────────── */
     .sdk-score.parity-high { color: #3ECF8E; }
     .sdk-score.parity-mid  { color: #f59e0b; }
@@ -457,9 +448,21 @@ export function renderHtml(
       <h1 class="site-title"><span>Supabase</span> SDK Capability Matrix</h1>
       <p class="build-info">Updated ${esc(buildDate)} · <a href="compliance.json">compliance.json</a></p>
     </div>
-    <div class="overall-badge">
-      <span class="label">Overall parity</span>
-      <span class="value">${pct(clamp01(parity.overall))}</span>
+  </div>
+  <div class="metrics-row">
+    <div class="metric-card">
+      <div class="metric-top">
+        <span class="label">Overall parity</span>
+        <span class="value">${pct(clamp01(parity.overall))}</span>
+      </div>
+      <p class="metric-desc">% of tracked features fully implemented in every core SDK (JavaScript, Flutter, Python, Swift) — a feature only counts if none of them are missing or partial.</p>
+    </div>
+    <div class="metric-card">
+      <div class="metric-top">
+        <span class="label">Coverage scope</span>
+        <span class="value">${pct(clamp01(parity.coverageScope))}</span>
+      </div>
+      <p class="metric-desc">Of the features marked implemented/partial in a core SDK, % that have a registered symbols list — real code evidence behind the "done" claim.</p>
     </div>
   </div>
   <div class="sdk-grid">

scripts/capability-matrix/src/report.ts

@@ -1,4 +1,4 @@
-import { LANGUAGES } from "./types.js";
+import { CORE_LANGUAGES, LANGUAGES } from "./types.js";
 import type { ComplianceMap, Language, LoadedArea, ParityReport } from "./types.js";
 
 export type { ParityReport };
@@ -10,40 +10,59 @@ export function computeParity(
   loaded: LoadedArea[],
   compliance: Partial<Record<Language, ComplianceMap>>
 ): ParityReport {
-  const featureParities: number[] = [];
-  const perAreaParities: Record<string, number[]> = {};
-  const perFeature: Record<string, number> = {};
+  const featurePasses: number[] = [];
+  const perAreaPasses: Record<string, number[]> = {};
   const langImplemented = Object.fromEntries(LANGUAGES.map((l) => [l, 0])) as Record<Language, number>;
   const langApplicable = Object.fromEntries(LANGUAGES.map((l) => [l, 0])) as Record<Language, number>;
+  let doneCells = 0;
+  let doneCellsWithSymbols = 0;
 
   for (const { area } of loaded) {
-    perAreaParities[area.area] ??= [];
+    perAreaPasses[area.area] ??= [];
     for (const feature of area.features ?? []) {
-      let implemented = 0;
-      let applicable = 0;
+      // Strict cross-SDK pass check (core languages only)
+      const applicableCore = CORE_LANGUAGES.filter(
+        (lang) => (compliance[lang]?.[feature.id]?.status ?? "not_implemented") !== "not_applicable"
+      );
+      const passes =
+        applicableCore.length > 0 &&
+        applicableCore.every((lang) => compliance[lang]?.[feature.id]?.status === "implemented");
+      const passValue = passes ? 1 : 0;
+      featurePasses.push(passValue);
+      perAreaPasses[area.area].push(passValue);
+
+      // Per-language completion score (all 7 langs). Only exactly `implemented`
+      // counts — partial claims aren't credited as done.
       for (const lang of LANGUAGES) {
         const status = compliance[lang]?.[feature.id]?.status ?? "not_implemented";
         if (status === "not_applicable") continue;
-        applicable++;
         langApplicable[lang]++;
-        if (status === "implemented" || status === "partially_implemented") {
-          implemented++;
-          langImplemented[lang]++;
-        }
+        if (status === "implemented") langImplemented[lang]++;
+      }
+
+      // Coverage scope (core languages only). Same rule — only exactly
+      // `implemented` counts as done.
+      for (const lang of CORE_LANGUAGES) {
+        const entry = compliance[lang]?.[feature.id];
+        const status = entry?.status ?? "not_implemented";
+        if (status !== "implemented") continue;
+        doneCells++;
+        if (entry?.symbols && entry.symbols.length > 0) doneCellsWithSymbols++;
       }
-      const parity = applicable === 0 ? 1 : implemented / applicable;
-      featureParities.push(parity);
-      perAreaParities[area.area].push(parity);
-      perFeature[feature.id] = parity;
     }
   }
 
   const perArea: Record<string, number> = {};
-  for (const [name, xs] of Object.entries(perAreaParities)) perArea[name] = mean(xs);
+  for (const [name, xs] of Object.entries(perAreaPasses)) perArea[name] = mean(xs);
 
   const perLanguage = Object.fromEntries(
     LANGUAGES.map((l) => [l, langApplicable[l] === 0 ? 0 : langImplemented[l] / langApplicable[l]])
   ) as Record<Language, number>;
 
-  return { overall: mean(featureParities), perArea, perLanguage, perFeature };
+  return {
+    overall: mean(featurePasses),
+    perArea,
+    perLanguage,
+    coverageScope: doneCells === 0 ? 0 : doneCellsWithSymbols / doneCells,
+  };
 }

scripts/capability-matrix/src/types.ts

@@ -9,6 +9,13 @@ export const LANGUAGES = [
 ] as const;
 export type Language = (typeof LANGUAGES)[number];
 
+export const CORE_LANGUAGES = [
+  "javascript",
+  "flutter",
+  "python",
+  "swift",
+] as const satisfies readonly Language[];
+
 export const STATUSES = [
   "implemented",
   "partially_implemented",
@@ -61,8 +68,7 @@ export interface ParityReport {
   overall: number;
   perArea: Record<string, number>;
   perLanguage: Record<Language, number>;
-  /** Cross-language parity score per feature ID (0–1). */
-  perFeature: Record<string, number>;
+  coverageScope: number;
 }
 
 /** Shape of site/compliance.json — raw compliance data plus precomputed parity. */

scripts/capability-matrix/test/report.test.ts

@@ -10,49 +10,157 @@ function area(id: string, features: ReturnType<typeof feature>[]): LoadedArea {
   return { file: "auth.yaml", area: { area: id, title: "T", description: "d", features } };
 }
 
-function compliance(lang: Language, statuses: Record<string, string>): Partial<Record<Language, ComplianceMap>> {
-  const map: ComplianceMap = {};
-  for (const [id, status] of Object.entries(statuses)) {
-    map[id] = { status: status as never };
-  }
-  return { [lang]: map };
+function entry(status: string, symbols?: string[]) {
+  return symbols ? { status: status as never, symbols } : { status: status as never };
 }
 
-describe("computeParity", () => {
-  it("returns 0 when all features have no compliance data", () => {
+describe("computeParity — strict overall/perArea (core langs only)", () => {
+  it("returns 0 overall when all features have no compliance data", () => {
     const a = area("auth", [feature("auth.a"), feature("auth.b")]);
     const report = computeParity([a], {});
     expect(report.overall).toBe(0);
     expect(report.perLanguage.javascript).toBe(0);
   });
 
-  it("treats not_applicable as excluded from the denominator", () => {
+  it("passes a feature only when every core lang is exactly implemented", () => {
     const a = area("auth", [feature("auth.a")]);
-    // js: implemented, others: not_implemented (from empty map) -> 1/7
-    const c = compliance("javascript", { "auth.a": "implemented" });
+    const c: Partial<Record<Language, ComplianceMap>> = {
+      javascript: { "auth.a": entry("implemented") },
+      flutter: { "auth.a": entry("implemented") },
+      python: { "auth.a": entry("implemented") },
+      swift: { "auth.a": entry("implemented") },
+    };
+    const report = computeParity([a], c);
+    expect(report.overall).toBe(1);
+    expect(report.perArea.auth).toBe(1);
+  });
+
+  it("ignores csharp/go/kotlin status entirely for the strict pass check", () => {
+    const a = area("auth", [feature("auth.a")]);
+    const c: Partial<Record<Language, ComplianceMap>> = {
+      javascript: { "auth.a": entry("implemented") },
+      flutter: { "auth.a": entry("implemented") },
+      python: { "auth.a": entry("implemented") },
+      swift: { "auth.a": entry("implemented") },
+      csharp: { "auth.a": entry("not_implemented") },
+      go: { "auth.a": entry("not_implemented") },
+      kotlin: { "auth.a": entry("not_implemented") },
+    };
+    const report = computeParity([a], c);
+    expect(report.overall).toBe(1);
+    // but csharp/go/kotlin still get their own (low) completion score
+    expect(report.perLanguage.csharp).toBe(0);
+  });
+
+  it("fails a feature when a core lang is only partially_implemented", () => {
+    const a = area("auth", [feature("auth.a")]);
+    const c: Partial<Record<Language, ComplianceMap>> = {
+      javascript: { "auth.a": entry("implemented") },
+      flutter: { "auth.a": entry("implemented") },
+      python: { "auth.a": entry("implemented") },
+      swift: { "auth.a": entry("partially_implemented") },
+    };
+    const report = computeParity([a], c);
+    expect(report.overall).toBe(0);
+  });
+
+  it("excludes a core lang marked not_applicable from the pass check", () => {
+    const a = area("auth", [feature("auth.a")]);
+    const c: Partial<Record<Language, ComplianceMap>> = {
+      javascript: { "auth.a": entry("implemented") },
+      flutter: { "auth.a": entry("implemented") },
+      python: { "auth.a": entry("implemented") },
+      swift: { "auth.a": entry("not_applicable") },
+    };
+    const report = computeParity([a], c);
+    expect(report.overall).toBe(1);
+  });
+
+  it("fails a feature that is not_applicable in every core lang", () => {
+    const a = area("auth", [feature("auth.a")]);
+    const c: Partial<Record<Language, ComplianceMap>> = {
+      javascript: { "auth.a": entry("not_applicable") },
+      flutter: { "auth.a": entry("not_applicable") },
+      python: { "auth.a": entry("not_applicable") },
+      swift: { "auth.a": entry("not_applicable") },
+    };
+    const report = computeParity([a], c);
+    expect(report.overall).toBe(0);
+  });
+
+  it("computes overall as passing features / total features across a mixed fixture", () => {
+    const a = area("auth", [feature("auth.a"), feature("auth.b")]);
+    const c: Partial<Record<Language, ComplianceMap>> = {
+      javascript: { "auth.a": entry("implemented"), "auth.b": entry("implemented") },
+      flutter: { "auth.a": entry("implemented"), "auth.b": entry("not_implemented") },
+      python: { "auth.a": entry("implemented"), "auth.b": entry("implemented") },
+      swift: { "auth.a": entry("implemented"), "auth.b": entry("implemented") },
+    };
+    // auth.a passes (all core implemented), auth.b fails (flutter not_implemented) -> 1/2
+    const report = computeParity([a], c);
+    expect(report.overall).toBe(0.5);
+    expect(report.perArea.auth).toBe(0.5);
+  });
+});
+
+describe("computeParity — perLanguage (all 7 langs, exact implemented only)", () => {
+  it("treats not_applicable as excluded from that language's own denominator", () => {
+    const a = area("auth", [feature("auth.a")]);
+    const c: Partial<Record<Language, ComplianceMap>> = {
+      javascript: { "auth.a": entry("implemented") },
+    };
     const report = computeParity([a], c);
-    expect(report.overall).toBeCloseTo(1 / 7, 6);
     expect(report.perLanguage.javascript).toBeCloseTo(1, 6);
     expect(report.perLanguage.swift).toBeCloseTo(0, 6);
   });
 
-  it("excludes not_applicable from feature parity", () => {
+  it("does not credit partially_implemented for a language's own score", () => {
     const a = area("auth", [feature("auth.a")]);
     const c: Partial<Record<Language, ComplianceMap>> = {
-      javascript: { "auth.a": { status: "implemented" } },
-      flutter: { "auth.a": { status: "implemented" } },
-      go: { "auth.a": { status: "not_applicable" } },
+      javascript: { "auth.a": entry("partially_implemented") },
     };
-    // applicable: 6 (go excluded), implemented: 2 -> 2/6
     const report = computeParity([a], c);
-    expect(report.overall).toBeCloseTo(2 / 6, 6);
-    expect(report.perArea.auth).toBeCloseTo(2 / 6, 6);
+    expect(report.perLanguage.javascript).toBe(0);
+  });
+});
+
+describe("computeParity — coverageScope", () => {
+  it("is 0 when there are no done cells at all", () => {
+    const a = area("auth", [feature("auth.a")]);
+    const report = computeParity([a], {});
+    expect(report.coverageScope).toBe(0);
   });
 
-  it("counts partially_implemented as implemented for parity", () => {
+  it("is the fraction of done core-lang cells that have symbols listed", () => {
     const a = area("auth", [feature("auth.a")]);
-    const c = compliance("javascript", { "auth.a": "partially_implemented" });
+    const c: Partial<Record<Language, ComplianceMap>> = {
+      javascript: { "auth.a": entry("implemented", ["Foo.bar"]) },
+      flutter: { "auth.a": entry("implemented") }, // done, no symbols
+      python: { "auth.a": entry("not_implemented") }, // not done, doesn't count
+      swift: { "auth.a": entry("partially_implemented", ["Bar.baz"]) }, // partial doesn't count as done
+    };
+    // done cells: javascript, flutter (2); with symbols: javascript (1) -> 1/2
     const report = computeParity([a], c);
-    expect(report.perLanguage.javascript).toBeCloseTo(1, 6);
+    expect(report.coverageScope).toBeCloseTo(1 / 2, 6);
+  });
+
+  it("excludes partially_implemented entirely from coverage scope, even with symbols", () => {
+    const a = area("auth", [feature("auth.a")]);
+    const c: Partial<Record<Language, ComplianceMap>> = {
+      javascript: { "auth.a": entry("partially_implemented", ["Foo.bar"]) },
+    };
+    const report = computeParity([a], c);
+    expect(report.coverageScope).toBe(0);
+  });
+
+  it("ignores csharp/go/kotlin cells even if they have symbols", () => {
+    const a = area("auth", [feature("auth.a")]);
+    const c: Partial<Record<Language, ComplianceMap>> = {
+      javascript: { "auth.a": entry("implemented") }, // done, no symbols
+      csharp: { "auth.a": entry("implemented", ["Csharp.Foo"]) },
+    };
+    // only javascript counts as a core done cell -> 0/1
+    const report = computeParity([a], c);
+    expect(report.coverageScope).toBe(0);
   });
 });