JSONbored · loopover-orb · Jul 29, 2026 · Jul 29, 2026
@@ -457,13 +457,17 @@
                     "precision": {
                       "type": "number",
                       "nullable": true
+                    },
+                    "unrecognized": {
+                      "type": "number"
                     }
                   },
                   "required": [
                     "ruleId",
                     "decided",
                     "confirmed",
-                    "precision"
+                    "precision",
+                    "unrecognized"
                   ]
                 }
               },

@@ -21,7 +21,9 @@ import { z } from "zod";
  */
 export const PublicRulePrecisionSchema = z.object({
   windowDays: z.number(),
-  rules: z.array(z.object({ ruleId: z.string(), decided: z.number(), confirmed: z.number(), precision: z.number().nullable() })),
+  rules: z.array(
+    z.object({ ruleId: z.string(), decided: z.number(), confirmed: z.number(), precision: z.number().nullable(), unrecognized: z.number() }),
+  ),
   reversals: z.object({ reopened: z.number(), reverted: z.number(), superseded: z.number() }),
   latestBacktestRun: z.object({ corpusChecksum: z.string(), at: z.string() }).nullable(),
 });

@@ -41,6 +41,10 @@ export type PublicRulePrecisionRow = {
   confirmed: number;
   /** confirmed / decided, rounded to 3 decimals; null below {@link PUBLIC_PRECISION_MIN_DECIDED}. */
   precision: number | null;
+  /** Overrides whose `$.verdict` was missing or neither `'reversed'` nor `'confirmed'` -- excluded from
+   *  `decided` and `confirmed` (and so never inflates `precision` or clears the sample floor), but surfaced
+   *  as a data-quality signal rather than silently dropped. */
+  unrecognized: number;
 };
 
 export type PublicRulePrecision = {
@@ -79,10 +83,12 @@ export const NON_ATTRIBUTABLE_OVERRIDE_PROVENANCES = ["slop_replay_backfill_v1"]
 export async function loadPublicRulePrecision(env: Env, nowMs: number = Date.now()): Promise<PublicRulePrecision> {
   const sinceIso = new Date(nowMs - PUBLIC_PRECISION_WINDOW_DAYS * 24 * 60 * 60 * 1000).toISOString();
 
-  const overrideRows = await safeAll<{ rule_id: string; decided: number; reversed: number }>(
+  const overrideRows = await safeAll<{ rule_id: string; confirmed: number; reversed: number; unrecognized: number }>(
     env,
-    `SELECT substr(event_type, ${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX.length + 1}) AS rule_id, COUNT(*) AS decided,
-            SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed
+    `SELECT substr(event_type, ${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX.length + 1}) AS rule_id,
+            SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'confirmed' THEN 1 ELSE 0 END) AS confirmed,
+            SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed,
+            SUM(CASE WHEN json_extract(metadata_json, '$.verdict') NOT IN ('confirmed', 'reversed') OR json_extract(metadata_json, '$.verdict') IS NULL THEN 1 ELSE 0 END) AS unrecognized
        FROM audit_events
       WHERE event_type LIKE '${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX}%' AND created_at >= ?
         AND COALESCE(json_extract(metadata_json, '$.provenance'), '') NOT IN (${NON_ATTRIBUTABLE_OVERRIDE_PROVENANCES.map((tag) => `'${tag}'`).join(", ")})
@@ -91,15 +97,19 @@ export async function loadPublicRulePrecision(env: Env, nowMs: number = Date.now
   );
   const rules: PublicRulePrecisionRow[] = overrideRows
     .map((row) => {
-      /* v8 ignore next 2 -- SUM(CASE) over a GROUP BY always yields a defined integer; the ?? guards a
+      /* v8 ignore next -- SUM(CASE) over a GROUP BY always yields a defined integer; the ?? guards a
        * future query-shape change, mirroring loadOverrideDayRows' identical note. */
+      const confirmed = row.confirmed ?? 0;
+      /* v8 ignore next */
       const reversed = row.reversed ?? 0;
-      const decided = row.decided;
-      const confirmed = decided - reversed;
+      /* v8 ignore next */
+      const unrecognized = row.unrecognized ?? 0;
+      const decided = confirmed + reversed;
       return {
         ruleId: row.rule_id,
         decided,
         confirmed,
+        unrecognized,
         precision: decided >= PUBLIC_PRECISION_MIN_DECIDED ? Math.round((confirmed / decided) * 1000) / 1000 : null,
       };
     })

@@ -33,6 +33,9 @@ export type CalibrationRuleTrendWeek = {
   confirmed: number | null;
   reversed: number | null;
   precisionPct: number | null;
+  /** Overrides whose `$.verdict` was missing or neither `'reversed'` nor `'confirmed'` -- a data-quality
+   *  signal, always visible regardless of the publication floor (never folded into `confirmed`). */
+  unrecognized: number;
 };
 
 export type CalibrationRuleTrend = { ruleId: string; weeks: CalibrationRuleTrendWeek[] };
@@ -51,7 +54,7 @@ export type CalibrationTrendReport = {
 };
 
 export type FiredDayRow = { ruleId: string; day: string; fired: number };
-export type OverrideDayRow = { ruleId: string; day: string; confirmed: number; reversed: number };
+export type OverrideDayRow = { ruleId: string; day: string; confirmed: number; reversed: number; unrecognized: number };
 export type BacktestRunDayRow = { day: string; regressed: number; improved: number; unchanged: number };
 
 const MS_PER_WEEK = 7 * 86_400_000;
@@ -84,11 +87,11 @@ export function buildCalibrationTrend(
   const currentStartMs = Date.parse(isoWeekStart(nowMs));
   const oldestStartMs = currentStartMs - (weeks - 1) * MS_PER_WEEK;
 
-  const ruleBuckets = new Map<string, Array<{ fired: number; confirmed: number; reversed: number }>>();
+  const ruleBuckets = new Map<string, Array<{ fired: number; confirmed: number; reversed: number; unrecognized: number }>>();
   const bucketsFor = (ruleId: string) => {
     const existing = ruleBuckets.get(ruleId);
     if (existing) return existing;
-    const created = Array.from({ length: weeks }, () => ({ fired: 0, confirmed: 0, reversed: 0 }));
+    const created = Array.from({ length: weeks }, () => ({ fired: 0, confirmed: 0, reversed: 0, unrecognized: 0 }));
     ruleBuckets.set(ruleId, created);
     return created;
   };
@@ -103,6 +106,7 @@ export function buildCalibrationTrend(
     const bucket = bucketsFor(row.ruleId)[offset]!;
     bucket.confirmed += row.confirmed;
     bucket.reversed += row.reversed;
+    bucket.unrecognized += row.unrecognized;
   }
 
   const runBuckets = Array.from({ length: weeks }, () => ({ regressed: 0, improved: 0, unchanged: 0 }));
@@ -128,6 +132,7 @@ export function buildCalibrationTrend(
           confirmed: publishable ? bucket.confirmed : null,
           reversed: publishable ? bucket.reversed : null,
           precisionPct: publishable ? roundPct(bucket.confirmed / decided) : null,
+          unrecognized: bucket.unrecognized,
         };
       }),
     }));
@@ -161,19 +166,28 @@ async function loadFiredDayRows(env: Env, sinceIso: string): Promise<FiredDayRow
  *  recordHumanOverride writes it) — bucketed by the override's OWN created_at: this trend reports how humans
  *  are judging a rule's calls per decision week (see the module doc's precision-semantics note). */
 async function loadOverrideDayRows(env: Env, sinceIso: string): Promise<OverrideDayRow[]> {
-  const rows = await safeAll<{ rule_id: string; day: string; confirmed: number; reversed: number }>(
+  const rows = await safeAll<{ rule_id: string; day: string; confirmed: number; reversed: number; unrecognized: number }>(
     env,
     `SELECT substr(event_type, ${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX.length + 1}) AS rule_id, date(created_at) AS day,
-            SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 0 ELSE 1 END) AS confirmed,
-            SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed
+            SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'confirmed' THEN 1 ELSE 0 END) AS confirmed,
+            SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed,
+            SUM(CASE WHEN json_extract(metadata_json, '$.verdict') NOT IN ('confirmed', 'reversed') OR json_extract(metadata_json, '$.verdict') IS NULL THEN 1 ELSE 0 END) AS unrecognized
        FROM audit_events
       WHERE event_type LIKE '${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX}%' AND created_at >= ?
       GROUP BY rule_id, day`,
     sinceIso,
   );
-  /* v8 ignore next 2 -- SUM(CASE ...) over a GROUP BY always yields a defined integer, never SQL NULL; the ?? 0
-   * fallbacks guard a future query-shape change, mirroring loadOrbDayRows' identical note. */
-  return rows.map((row) => ({ ruleId: row.rule_id, day: row.day, confirmed: row.confirmed ?? 0, reversed: row.reversed ?? 0 }));
+  return rows.map((row) => ({
+    ruleId: row.rule_id,
+    day: row.day,
+    /* v8 ignore next -- SUM(CASE ...) over a GROUP BY always yields a defined integer, never SQL NULL; the
+     * ?? 0 fallbacks guard a future query-shape change, mirroring loadOrbDayRows' identical note. */
+    confirmed: row.confirmed ?? 0,
+    /* v8 ignore next */
+    reversed: row.reversed ?? 0,
+    /* v8 ignore next */
+    unrecognized: row.unrecognized ?? 0,
+  }));
 }
 
 /** Day-bucketed backtest runs across BOTH sibling event types, verdict read from the persisted

@@ -21,8 +21,8 @@ const ISSUED_AT = "2026-07-27T12:00:00.000Z";
 const PRECISION_WITH_FREEZE_POINT: PublicRulePrecision = {
   windowDays: 90,
   rules: [
-    { ruleId: "ai_consensus_defect", decided: 25, confirmed: 20, precision: 0.8 },
-    { ruleId: "sparse_rule", decided: 9, confirmed: 9, precision: null },
+    { ruleId: "ai_consensus_defect", decided: 25, confirmed: 20, precision: 0.8, unrecognized: 0 },
+    { ruleId: "sparse_rule", decided: 9, confirmed: 9, precision: null, unrecognized: 0 },
   ],
   reversals: { reopened: 2, reverted: 1, superseded: 3 },
   latestBacktestRun: { corpusChecksum: "abc123def456", at: "2026-07-27T10:00:00.000Z" },
@@ -99,7 +99,7 @@ describe("buildEvalScoreRecordsFromRulePrecision (#9266)", () => {
     // than a hardcoded constant. Previously EVERY record published null, including the 25-decided one above,
     // so a validator re-deriving decided/(decided+abstained) per #9215 computed 1 and disagreed with the field.
     const records = await buildEvalScoreRecordsFromRulePrecision(
-      { ...PRECISION_WITH_FREEZE_POINT, rules: [{ ruleId: "never_fired", decided: 0, confirmed: 0, precision: null }] },
+      { ...PRECISION_WITH_FREEZE_POINT, rules: [{ ruleId: "never_fired", decided: 0, confirmed: 0, precision: null, unrecognized: 0 }] },
       ISSUED_AT,
     );
     expect(records).toHaveLength(1);
@@ -207,8 +207,7 @@ describe("per-rule corpus commitments when no backtest run is persisted (#9805)"
     reversals: { reopened: 0, reverted: 0, superseded: 0 },
     latestBacktestRun,
   });
-  const rule = (ruleId: string): PublicRulePrecision["rules"][number] =>
-    ({ ruleId, decided: 40, confirmed: 25, precision: 0.625 }) as PublicRulePrecision["rules"][number];
+  const rule = (ruleId: string): PublicRulePrecision["rules"][number] => ({ ruleId, decided: 40, confirmed: 25, precision: 0.625, unrecognized: 0 });
 
   it("REGRESSION: publishes a record per rule instead of [], committing to that rule's published corpus", async () => {
     const records = await buildEvalScoreRecordsFromRulePrecision(

@@ -38,8 +38,8 @@ describe("loadPublicRulePrecision (#8230)", () => {
     const block = await loadPublicRulePrecision(env, NOW);
     expect(block.windowDays).toBe(PUBLIC_PRECISION_WINDOW_DAYS);
     expect(block.rules).toEqual([
-      { ruleId: "ai_consensus_defect", decided: 25, confirmed: 20, precision: 0.8 },
-      { ruleId: "linked_issue_scope_mismatch", decided: 12, confirmed: 9, precision: 0.75 },
+      { ruleId: "ai_consensus_defect", decided: 25, confirmed: 20, precision: 0.8, unrecognized: 0 },
+      { ruleId: "linked_issue_scope_mismatch", decided: 12, confirmed: 9, precision: 0.75, unrecognized: 0 },
     ]);
   });
 
@@ -55,7 +55,9 @@ describe("loadPublicRulePrecision (#8230)", () => {
     });
 
     const block = await loadPublicRulePrecision(env, NOW);
-    expect(block.rules).toEqual([{ ruleId: "sparse_rule", decided: PUBLIC_PRECISION_MIN_DECIDED - 1, confirmed: PUBLIC_PRECISION_MIN_DECIDED - 1, precision: null }]);
+    expect(block.rules).toEqual([
+      { ruleId: "sparse_rule", decided: PUBLIC_PRECISION_MIN_DECIDED - 1, confirmed: PUBLIC_PRECISION_MIN_DECIDED - 1, precision: null, unrecognized: 0 },
+    ]);
   });
 
   it("REGRESSION: excludes counterfactual-replay rows whose label came from a DIFFERENT rule's human verdicts", async () => {
@@ -76,7 +78,7 @@ describe("loadPublicRulePrecision (#8230)", () => {
     }
 
     const block = await loadPublicRulePrecision(env, NOW);
-    expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 20, confirmed: 15, precision: 0.75 }]);
+    expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 20, confirmed: 15, precision: 0.75, unrecognized: 0 }]);
   });
 
   it("keeps synthesized rows whose labels ARE about the rule they are filed under", async () => {
@@ -95,7 +97,30 @@ describe("loadPublicRulePrecision (#8230)", () => {
     }
 
     const block = await loadPublicRulePrecision(env, NOW);
-    expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 12, confirmed: 9, precision: 0.75 }]);
+    expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 12, confirmed: 9, precision: 0.75, unrecognized: 0 }]);
+  });
+
+  it("REGRESSION (#9640): a missing or unrecognized $.verdict is counted as unrecognized, never folded into confirmed", async () => {
+    // Before the fix, `decided` was COUNT(*) and `confirmed = decided - reversed`, so any row whose verdict
+    // was absent or neither 'reversed' nor 'confirmed' inflated confirmed AND cleared the sample floor: this
+    // would have reported decided: 15, confirmed: 14, precision: 0.933.
+    const env = createTestEnv();
+    await seedVerdicts(env, "linked_issue_scope_mismatch", 9, 1);
+    for (let i = 0; i < 5; i += 1) {
+      await recordAuditEvent(env, {
+        eventType: "signal.human_override:linked_issue_scope_mismatch",
+        actor: "human",
+        targetKey: `acme/widgets#${i + 100}`,
+        outcome: "completed",
+        metadata: {},
+        createdAt: new Date(NOW - 2000 - i).toISOString(),
+      });
+    }
+
+    const block = await loadPublicRulePrecision(env, NOW);
+    expect(block.rules).toEqual([
+      { ruleId: "linked_issue_scope_mismatch", decided: 10, confirmed: 9, precision: 0.9, unrecognized: 5 },
+    ]);
   });
 
   it("counts all three reversal shapes over the window and surfaces the latest backtest run's corpus checksum", async () => {
@@ -156,7 +181,7 @@ describe("loadPublicRulePrecision (#8230)", () => {
     expect(block.latestBacktestRun).toBeNull();
     // The scores come from a different dataset (human-override events) and are unaffected -- an empty corpus
     // means the numbers are uncommitted, never that they are zero.
-    expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 20, confirmed: 15, precision: 0.75 }]);
+    expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 20, confirmed: 15, precision: 0.75, unrecognized: 0 }]);
   });
 
   it("EMPTY_CORPUS_CHECKSUM is the exporter's own checksum over zero cases", async () => {

@@ -25,30 +25,57 @@ describe("buildCalibrationTrend (#8113)", () => {
       // A second day in the SAME week — must accumulate.
       { ruleId: "linked_issue_scope_mismatch", day: priorMonday, fired: 2 },
     ];
-    const overrides: OverrideDayRow[] = [{ ruleId: "linked_issue_scope_mismatch", day: priorMonday, confirmed: 3, reversed: 1 }];
+    const overrides: OverrideDayRow[] = [
+      { ruleId: "linked_issue_scope_mismatch", day: priorMonday, confirmed: 3, reversed: 1, unrecognized: 0 },
+    ];
     const trend = buildCalibrationTrend(fired, overrides, [], NOW, 2);
     expect(trend.rules).toHaveLength(1);
     const [rule] = trend.rules;
     expect(rule!.ruleId).toBe("linked_issue_scope_mismatch");
     expect(rule!.weeks).toEqual([
-      { weekStart: priorMonday, fired: 6, confirmed: 3, reversed: 1, precisionPct: 75 },
-      { weekStart: currentMonday, fired: 0, confirmed: null, reversed: null, precisionPct: null },
+      { weekStart: priorMonday, fired: 6, confirmed: 3, reversed: 1, precisionPct: 75, unrecognized: 0 },
+      { weekStart: currentMonday, fired: 0, confirmed: null, reversed: null, precisionPct: null, unrecognized: 0 },
     ]);
   });
 
   it("keeps a week's verdict split null below MIN_CALIBRATION_TREND_SAMPLE decided — unknown never fakes 0 or 100", () => {
-    const overrides: OverrideDayRow[] = [{ ruleId: "duplicate_pr_risk", day: currentMonday, confirmed: MIN_CALIBRATION_TREND_SAMPLE - 1, reversed: 0 }];
+    const overrides: OverrideDayRow[] = [
+      { ruleId: "duplicate_pr_risk", day: currentMonday, confirmed: MIN_CALIBRATION_TREND_SAMPLE - 1, reversed: 0, unrecognized: 0 },
+    ];
     const trend = buildCalibrationTrend([], overrides, [], NOW, 1);
-    expect(trend.rules[0]!.weeks[0]).toEqual({ weekStart: currentMonday, fired: 0, confirmed: null, reversed: null, precisionPct: null });
+    expect(trend.rules[0]!.weeks[0]).toEqual({
+      weekStart: currentMonday,
+      fired: 0,
+      confirmed: null,
+      reversed: null,
+      precisionPct: null,
+      unrecognized: 0,
+    });
   });
 
   it("creates a rule bucket from an override-only history (no firings recorded in the window)", () => {
-    const overrides: OverrideDayRow[] = [{ ruleId: "missing_linked_issue", day: currentMonday, confirmed: 2, reversed: 2 }];
+    const overrides: OverrideDayRow[] = [{ ruleId: "missing_linked_issue", day: currentMonday, confirmed: 2, reversed: 2, unrecognized: 0 }];
     const trend = buildCalibrationTrend([], overrides, [], NOW, 1);
     expect(trend.rules[0]!.weeks[0]!.precisionPct).toBe(50);
     expect(trend.rules[0]!.weeks[0]!.fired).toBe(0);
   });
 
+  it("REGRESSION (#9640): counts a missing/unrecognized $.verdict as unrecognized, excluded from decided/confirmed", () => {
+    const overrides: OverrideDayRow[] = [
+      { ruleId: "linked_issue_scope_mismatch", day: currentMonday, confirmed: 9, reversed: 1, unrecognized: 5 },
+    ];
+    const trend = buildCalibrationTrend([], overrides, [], NOW, 1);
+    // Below the fix, decided would have been 15 (confirmed folded the 5 unrecognized rows in) and precision 0.933.
+    expect(trend.rules[0]!.weeks[0]).toEqual({
+      weekStart: currentMonday,
+      fired: 0,
+      confirmed: 9,
+      reversed: 1,
+      precisionPct: 90,
+      unrecognized: 5,
+    });
+  });
+
   it("sorts rules by ruleId for byte-stable output", () => {
     const fired: FiredDayRow[] = [
       { ruleId: "zeta_rule", day: currentMonday, fired: 1 },
@@ -64,7 +91,7 @@ describe("buildCalibrationTrend (#8113)", () => {
       { ruleId: "r", day: outside, fired: 5 },
       { ruleId: "r", day: "not-a-day", fired: 5 },
     ];
-    const overrides: OverrideDayRow[] = [{ ruleId: "r", day: future, confirmed: 5, reversed: 5 }];
+    const overrides: OverrideDayRow[] = [{ ruleId: "r", day: future, confirmed: 5, reversed: 5, unrecognized: 0 }];
     const runs: BacktestRunDayRow[] = [{ day: "junk", regressed: 1, improved: 1, unchanged: 1 }];
     const trend = buildCalibrationTrend(fired, overrides, runs, NOW, 2);
     expect(trend.rules).toEqual([]);
@@ -154,7 +181,7 @@ describe("loadCalibrationTrend (#8113)", () => {
     const trend = await loadCalibrationTrend(env, NOW);
     const rule = trend.rules.find((entry) => entry.ruleId === "linked_issue_scope_mismatch");
     const priorWeek = rule!.weeks.find((week) => week.weekStart === priorMonday);
-    expect(priorWeek).toEqual({ weekStart: priorMonday, fired: 1, confirmed: 3, reversed: 1, precisionPct: 75 });
+    expect(priorWeek).toEqual({ weekStart: priorMonday, fired: 1, confirmed: 3, reversed: 1, precisionPct: 75, unrecognized: 0 });
     const runWeek = trend.backtestRuns.find((week) => week.weekStart === priorMonday);
     expect(runWeek).toEqual({ weekStart: priorMonday, runs: 3, regressed: 1, improved: 1, unchanged: 1 });
   });