fix(runner): quality tiers that measured nothing WARN instead of false-green PASS

CHANGELOG.md

@@ -23,6 +23,11 @@ follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
   aggregate report with a missing/unrecognized gate decision, previously defaulted the merge check
   to `PASS` — a broken gate could post a green check and unblock a merge. Both now resolve to
   `BLOCK`; only an explicit, recognized `PASS`/`WARN`/`BLOCK` is trusted.
+- **Quality tiers that ran but measured nothing now `WARN` instead of `PASS`.** "0 failures" is not
+  "no measurements": the perf/Lighthouse tier when routes were audited but no metric was readable,
+  the k6 load tier when the run issued zero requests, the component tier when the runner collected
+  0 tests, the Pact tier when no contracts were verified, the i18n gate when no locales were
+  compared, and the CUJ gate when it's enabled but no CUJ definitions loaded.
 
 ## [0.4.0] — 2026-07-10 · "Dogfood"
 

packages/cli/src/cuj-gate.test.ts

@@ -79,6 +79,15 @@ describe('evaluateCujGateForRun', () => {
     expect(outcome.reports[0]!.status).toBe('BROKEN');
   });
 
+  it('WARNs when the gate is enabled but no CUJ definitions loaded', async () => {
+    const outcome = await evaluateCujGateForRun([result('TC-pay', 'PASS')], enabledCfg(), {
+      source: memSource({}),
+      changeSurface: surface(),
+    });
+    expect(outcome.gate.decision).toBe('WARN');
+    expect(outcome.gate.reason).toMatch(/no CUJ definitions/i);
+  });
+
   it('is a neutral PASS when the change touches no journey', async () => {
     const outcome = await evaluateCujGateForRun([result('TC-pay', 'FAIL')], enabledCfg(), {
       source: memSource({ 'checkout.yaml': CHECKOUT_DEF }),

packages/cli/src/cuj-gate.ts

@@ -83,7 +83,17 @@ export async function evaluateCujGateForRun(
 
   const { cujs, errors } = await new CujRegistry(run.source, run.parse).load(cfg.cuj.dir);
   for (const err of errors) logger.warn(`CUJ definition skipped: ${err.message}`);
-  if (cujs.length === 0) return { gate: NEUTRAL, reports: [] };
+  // The gate is enabled but no CUJ definitions loaded (missing/misconfigured `cuj.dir`, or every
+  // def was malformed). It ran but measured nothing — WARN rather than a confident neutral PASS.
+  if (cujs.length === 0) {
+    return {
+      gate: {
+        decision: 'WARN',
+        reason: `CUJ gate is enabled but no CUJ definitions were loaded from '${cfg.cuj.dir}' — journey health was not measured.`,
+      },
+      reports: [],
+    };
+  }
 
   const touched = resolveTouchedCujs(run.changeSurface, cujs);
   if (touched.length === 0) return { gate: NEUTRAL, reports: [] };

packages/runner/src/api/pact.test.ts

@@ -159,6 +159,16 @@ describe('evaluatePactGate', () => {
     expect(evaluatePactGate(results).decision).toBe('PASS');
   });
 
+  it('WARNs when no interactions were verified at all (no contracts found)', () => {
+    expect(evaluatePactGate([]).decision).toBe('WARN');
+    const withEmptyChecks: ContractVerificationResult[] = [
+      { consumer: 'web-app', provider: 'checkout-service', checks: [] },
+    ];
+    const gate = evaluatePactGate(withEmptyChecks);
+    expect(gate.decision).toBe('WARN');
+    expect(gate.reason).toMatch(/no.*contract/i);
+  });
+
   it('BLOCKs when any interaction check failed', () => {
     const results: ContractVerificationResult[] = [
       {

packages/runner/src/api/pact.ts

@@ -161,16 +161,27 @@ export function pactVerificationToCtrf(results: ContractVerificationResult[]): C
 
 /**
  * Pure gate mapping over {@link ContractVerificationResult}s. Any failed interaction check →
- * `BLOCK` (a broken contract is a breaking change for a live consumer); otherwise `PASS`.
+ * `BLOCK` (a broken contract is a breaking change for a live consumer); zero interactions verified
+ * at all (no contracts found) → `WARN` (a config/broker gap, not a clean bill); otherwise `PASS`.
  */
 export function evaluatePactGate(results: ContractVerificationResult[]): GateDecision {
-  const failed = results.flatMap((r) => r.checks).filter((c) => !c.success);
+  const checks = results.flatMap((r) => r.checks);
+  const failed = checks.filter((c) => !c.success);
   if (failed.length > 0) {
     return {
       decision: 'BLOCK',
       reason: `${failed.length} contract interaction(s) failed verification`,
     };
   }
+  // No interactions were verified at all (broker returned no contracts, wrong provider name, or a
+  // tag that matched nothing). "0 verified" is a broker/config gap, not "all verified".
+  if (checks.length === 0) {
+    return {
+      decision: 'WARN',
+      reason:
+        'no consumer contract interactions were verified (no contracts found — broker/tag misconfiguration?)',
+    };
+  }
   return { decision: 'PASS', reason: 'all contract interactions verified' };
 }
 

packages/runner/src/component/component.test.ts

@@ -73,7 +73,9 @@ describe('evaluateComponentGate', () => {
     expect(gate.decision).toBe('PASS');
   });
 
-  it('PASSes on an empty report', () => {
-    expect(evaluateComponentGate(componentResultsToCtrf([])).decision).toBe('PASS');
+  it('WARNs on an empty report — 0 tests collected is "did not measure", not "no failures"', () => {
+    const gate = evaluateComponentGate(componentResultsToCtrf([]));
+    expect(gate.decision).toBe('WARN');
+    expect(gate.reason).toMatch(/0 tests/i);
   });
 });

packages/runner/src/component/component.ts

@@ -67,8 +67,8 @@ export function componentResultsToCtrf(results: ComponentTestResult[]): CTRFRepo
 
 /**
  * Pure gate mapping over the CTRF output of {@link componentResultsToCtrf}: any failed component
- * test → `BLOCK`, otherwise `PASS`. Component tests have no "acceptable degradation" tier the way
- * a11y/perf budgets do, so there is no `WARN` outcome here.
+ * test → `BLOCK`; a run that collected zero tests (nothing matched/discovered) → `WARN` (it did
+ * not measure); otherwise `PASS`.
  */
 export function evaluateComponentGate(report: CTRFReport): GateDecision {
   const failed = report.results.tests.filter((t) => t.status === 'failed');
@@ -78,6 +78,14 @@ export function evaluateComponentGate(report: CTRFReport): GateDecision {
       reason: `${failed.length} component test failure(s)`,
     };
   }
+  // The runner ran but collected no tests (grep/testNamePattern matched nothing, or no
+  // component tests/stories were discovered). "0 tests" is "did not measure", not "no failures".
+  if (report.results.tests.length === 0) {
+    return {
+      decision: 'WARN',
+      reason: 'component tier ran but collected 0 tests (nothing matched or was discovered)',
+    };
+  }
   return { decision: 'PASS', reason: 'no component test failures' };
 }
 

packages/runner/src/i18n/i18n.test.ts

@@ -123,4 +123,16 @@ describe('evaluateI18nGate', () => {
     expect(evaluateI18nGate(emptyReport, { gate: 'warn' }).decision).toBe('PASS');
     expect(evaluateI18nGate(emptyReport, { gate: 'block' }).decision).toBe('PASS');
   });
+
+  it('WARNs when the check compared no locales (measurement gap, not a clean bill)', () => {
+    const gate = evaluateI18nGate(emptyReport, { gate: 'warn' }, { comparedLocaleCount: 0 });
+    expect(gate.decision).toBe('WARN');
+    expect(gate.reason).toMatch(/no locales|nothing was measured/i);
+  });
+
+  it('PASSes when locales were compared and nothing is missing', () => {
+    expect(
+      evaluateI18nGate(emptyReport, { gate: 'warn' }, { comparedLocaleCount: 2 }).decision,
+    ).toBe('PASS');
+  });
 });

packages/runner/src/i18n/i18n.ts

@@ -101,12 +101,29 @@ export interface I18nGateConfig {
  * Pure gate mapping over the CTRF output of {@link i18nResultsToCtrf}. i18n gaps rarely warrant
  * blocking a merge, so any missing translation → `WARN` by default; set `cfg.gate` to `'block'`
  * to treat gaps as blocking, or `'off'` to keep the check informational-only (always `PASS`).
+ *
+ * `findMissingTranslations` returns `[]` both when everything is translated AND when it could
+ * compare nothing (the default locale wasn't loaded, or there are no other locales) — the latter
+ * is a measurement gap, not a clean bill of health. Pass `measurement.comparedLocaleCount` so the
+ * gate can `WARN` on that case instead of a false `PASS`.
  */
-export function evaluateI18nGate(report: CTRFReport, cfg: I18nGateConfig): GateDecision {
+export function evaluateI18nGate(
+  report: CTRFReport,
+  cfg: I18nGateConfig,
+  measurement?: { comparedLocaleCount: number },
+): GateDecision {
   if (cfg.gate === 'off') {
     return { decision: 'PASS', reason: 'i18n gate is disabled (gate: "off")' };
   }
 
+  if (measurement && measurement.comparedLocaleCount === 0) {
+    return {
+      decision: 'WARN',
+      reason:
+        'i18n check compared no locales (default locale missing, or no other locales to compare) — nothing was measured',
+    };
+  }
+
   const failed = report.results.tests.filter((t) => t.status === 'failed').length;
   if (failed === 0) {
     return { decision: 'PASS', reason: 'no missing translations found' };

packages/runner/src/load/k6-load.test.ts

@@ -68,31 +68,37 @@ describe('k6LoadResultsToCtrf', () => {
 describe('evaluateLoadGate', () => {
   it('PASSes when the summary is within budget', () => {
     const report = k6LoadResultsToCtrf(withinBudget, thresholds);
-    expect(evaluateLoadGate(report)).toEqual({
+    expect(evaluateLoadGate(report, withinBudget)).toEqual({
       decision: 'PASS',
       reason: expect.any(String),
     });
   });
 
   it('BLOCKs when p95 latency is breached', () => {
-    const report = k6LoadResultsToCtrf({ ...withinBudget, p95Ms: 900 }, thresholds);
-    const gate = evaluateLoadGate(report);
+    const summary = { ...withinBudget, p95Ms: 900 };
+    const gate = evaluateLoadGate(k6LoadResultsToCtrf(summary, thresholds), summary);
     expect(gate.decision).toBe('BLOCK');
     expect(gate.reason).toContain('p95Ms');
   });
 
   it('BLOCKs when the error rate is breached', () => {
-    const report = k6LoadResultsToCtrf({ ...withinBudget, errorRate: 0.05 }, thresholds);
-    const gate = evaluateLoadGate(report);
+    const summary = { ...withinBudget, errorRate: 0.05 };
+    const gate = evaluateLoadGate(k6LoadResultsToCtrf(summary, thresholds), summary);
     expect(gate.decision).toBe('BLOCK');
     expect(gate.reason).toContain('errorRate');
   });
 
   it('BLOCKs and reports every breached threshold when multiple are breached', () => {
-    const report = k6LoadResultsToCtrf(breached, thresholds);
-    const gate = evaluateLoadGate(report);
+    const gate = evaluateLoadGate(k6LoadResultsToCtrf(breached, thresholds), breached);
     expect(gate.decision).toBe('BLOCK');
     expect(gate.reason).toContain('p95Ms');
     expect(gate.reason).toContain('errorRate');
   });
+
+  it('WARNs when the load run issued zero requests (measured nothing)', () => {
+    const zeroRequests: K6LoadSummary = { p95Ms: 0, p99Ms: 0, errorRate: 0, requests: 0 };
+    const gate = evaluateLoadGate(k6LoadResultsToCtrf(zeroRequests, thresholds), zeroRequests);
+    expect(gate.decision).toBe('WARN');
+    expect(gate.reason).toMatch(/zero requests/i);
+  });
 });

packages/runner/src/load/k6-load.ts

@@ -95,10 +95,19 @@ export function k6LoadResultsToCtrf(
 
 /**
  * Pure gate mapping over the CTRF output of {@link k6LoadResultsToCtrf}: any breached threshold
- * (p95, p99, or error rate) → `BLOCK`, otherwise `PASS`. Load thresholds have no "acceptable
- * degradation" tier the way a11y/perf budgets do, so there is no `WARN` outcome here.
+ * (p95, p99, or error rate) → `BLOCK`. A run that issued zero requests measured nothing (missing
+ * latency/error metrics coerce to 0 and spuriously pass every threshold) → `WARN`. Otherwise `PASS`.
  */
-export function evaluateLoadGate(report: CTRFReport): GateDecision {
+export function evaluateLoadGate(report: CTRFReport, summary: K6LoadSummary): GateDecision {
+  // Zero requests issued (script error, target unreachable) → the latency/error thresholds were
+  // never actually measured; they only "passed" because absent metrics normalize to 0.
+  if (summary.requests === 0) {
+    return {
+      decision: 'WARN',
+      reason:
+        'load test issued zero requests — latency and error-rate thresholds were not measured',
+    };
+  }
   const failed = report.results.tests.filter((t) => t.status === 'failed');
   if (failed.length > 0) {
     return {
@@ -197,6 +206,6 @@ export function runK6Load(
       requests: raw.metrics?.http_reqs?.values?.count ?? 0,
     };
     const report = k6LoadResultsToCtrf(summary, cfg.thresholds);
-    return { summary, report, gate: evaluateLoadGate(report) };
+    return { summary, report, gate: evaluateLoadGate(report, summary) };
   });
 }

packages/runner/src/perf/lighthouse.test.ts

@@ -143,4 +143,21 @@ describe('evaluatePerfBudgetGate', () => {
     const gate = evaluatePerfBudgetGate(lighthouseResultsToCtrf(results, budgets));
     expect(gate.decision).toBe('PASS');
   });
+
+  it('WARNs when routes were audited but no metrics could be measured', () => {
+    // A runtime-error report: no categories, no audits → every metric undefined → 0 CTRF tests.
+    const results: LighthouseRouteResult[] = [
+      { route: 'https://preview.example.com/checkout', report: {} },
+    ];
+    const report = lighthouseResultsToCtrf(results, budgets);
+    expect(report.results.tests).toHaveLength(0); // pins the "measured nothing" signal
+    const gate = evaluatePerfBudgetGate(report, results.length);
+    expect(gate.decision).toBe('WARN');
+    expect(gate.reason).toMatch(/no performance metrics/i);
+  });
+
+  it('PASSes when no routes were audited at all (nothing to measure)', () => {
+    const report = lighthouseResultsToCtrf([], budgets);
+    expect(evaluatePerfBudgetGate(report, 0).decision).toBe('PASS');
+  });
 });

packages/runner/src/perf/lighthouse.ts

@@ -162,10 +162,22 @@ export function lighthouseResultsToCtrf(
 
 /**
  * Pure gate mapping over the CTRF output of {@link lighthouseResultsToCtrf}: any `failed` metric
- * → `BLOCK`, any `near-budget` tag with no failures → `WARN`, else `PASS`.
+ * → `BLOCK`, any `near-budget` tag with no failures → `WARN`, else `PASS`. When `auditedRouteCount`
+ * routes were audited but no metric could be read (a runtime-error report), it is `WARN` — a
+ * measurement gap must not read as "all budgets satisfied".
  */
-export function evaluatePerfBudgetGate(report: CTRFReport): GateDecision {
+export function evaluatePerfBudgetGate(report: CTRFReport, auditedRouteCount = 0): GateDecision {
   const { tests } = report.results;
+
+  // Routes were audited but zero metrics were measurable (every Lighthouse metric was undefined,
+  // e.g. the page failed to load). "Measured nothing" is not "all budgets satisfied".
+  if (auditedRouteCount > 0 && tests.length === 0) {
+    return {
+      decision: 'WARN',
+      reason: `${auditedRouteCount} route(s) audited but no performance metrics were measured`,
+    };
+  }
+
   const failed = tests.filter((t) => t.status === 'failed');
   if (failed.length > 0) {
     return {
@@ -268,5 +280,5 @@ export async function runLighthouseAudit(
   }
 
   const report = lighthouseResultsToCtrf(results, budgets);
-  return { results, report, gate: evaluatePerfBudgetGate(report) };
+  return { results, report, gate: evaluatePerfBudgetGate(report, results.length) };
 }