diff --git a/CHANGELOG.md b/CHANGELOG.md index e907b8d..0d5027f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,11 @@ follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html). aggregate report with a missing/unrecognized gate decision, previously defaulted the merge check to `PASS` — a broken gate could post a green check and unblock a merge. Both now resolve to `BLOCK`; only an explicit, recognized `PASS`/`WARN`/`BLOCK` is trusted. +- **Quality tiers that ran but measured nothing now `WARN` instead of `PASS`.** "0 failures" is not + "no measurements": the perf/Lighthouse tier when routes were audited but no metric was readable, + the k6 load tier when the run issued zero requests, the component tier when the runner collected + 0 tests, the Pact tier when no contracts were verified, the i18n gate when no locales were + compared, and the CUJ gate when it's enabled but no CUJ definitions loaded. ## [0.4.0] — 2026-07-10 · "Dogfood" diff --git a/packages/cli/src/cuj-gate.test.ts b/packages/cli/src/cuj-gate.test.ts index 604bfbb..59a491d 100644 --- a/packages/cli/src/cuj-gate.test.ts +++ b/packages/cli/src/cuj-gate.test.ts @@ -79,6 +79,15 @@ describe('evaluateCujGateForRun', () => { expect(outcome.reports[0]!.status).toBe('BROKEN'); }); + it('WARNs when the gate is enabled but no CUJ definitions loaded', async () => { + const outcome = await evaluateCujGateForRun([result('TC-pay', 'PASS')], enabledCfg(), { + source: memSource({}), + changeSurface: surface(), + }); + expect(outcome.gate.decision).toBe('WARN'); + expect(outcome.gate.reason).toMatch(/no CUJ definitions/i); + }); + it('is a neutral PASS when the change touches no journey', async () => { const outcome = await evaluateCujGateForRun([result('TC-pay', 'FAIL')], enabledCfg(), { source: memSource({ 'checkout.yaml': CHECKOUT_DEF }), diff --git a/packages/cli/src/cuj-gate.ts b/packages/cli/src/cuj-gate.ts index 949a899..e6c6c72 100644 --- a/packages/cli/src/cuj-gate.ts +++ b/packages/cli/src/cuj-gate.ts @@ -83,7 +83,17 @@ export async function evaluateCujGateForRun( const { cujs, errors } = await new CujRegistry(run.source, run.parse).load(cfg.cuj.dir); for (const err of errors) logger.warn(`CUJ definition skipped: ${err.message}`); - if (cujs.length === 0) return { gate: NEUTRAL, reports: [] }; + // The gate is enabled but no CUJ definitions loaded (missing/misconfigured `cuj.dir`, or every + // def was malformed). It ran but measured nothing — WARN rather than a confident neutral PASS. + if (cujs.length === 0) { + return { + gate: { + decision: 'WARN', + reason: `CUJ gate is enabled but no CUJ definitions were loaded from '${cfg.cuj.dir}' — journey health was not measured.`, + }, + reports: [], + }; + } const touched = resolveTouchedCujs(run.changeSurface, cujs); if (touched.length === 0) return { gate: NEUTRAL, reports: [] }; diff --git a/packages/runner/src/api/pact.test.ts b/packages/runner/src/api/pact.test.ts index 89e8a2d..87a6427 100644 --- a/packages/runner/src/api/pact.test.ts +++ b/packages/runner/src/api/pact.test.ts @@ -159,6 +159,16 @@ describe('evaluatePactGate', () => { expect(evaluatePactGate(results).decision).toBe('PASS'); }); + it('WARNs when no interactions were verified at all (no contracts found)', () => { + expect(evaluatePactGate([]).decision).toBe('WARN'); + const withEmptyChecks: ContractVerificationResult[] = [ + { consumer: 'web-app', provider: 'checkout-service', checks: [] }, + ]; + const gate = evaluatePactGate(withEmptyChecks); + expect(gate.decision).toBe('WARN'); + expect(gate.reason).toMatch(/no.*contract/i); + }); + it('BLOCKs when any interaction check failed', () => { const results: ContractVerificationResult[] = [ { diff --git a/packages/runner/src/api/pact.ts b/packages/runner/src/api/pact.ts index 412c883..8efc545 100644 --- a/packages/runner/src/api/pact.ts +++ b/packages/runner/src/api/pact.ts @@ -161,16 +161,27 @@ export function pactVerificationToCtrf(results: ContractVerificationResult[]): C /** * Pure gate mapping over {@link ContractVerificationResult}s. Any failed interaction check → - * `BLOCK` (a broken contract is a breaking change for a live consumer); otherwise `PASS`. + * `BLOCK` (a broken contract is a breaking change for a live consumer); zero interactions verified + * at all (no contracts found) → `WARN` (a config/broker gap, not a clean bill); otherwise `PASS`. */ export function evaluatePactGate(results: ContractVerificationResult[]): GateDecision { - const failed = results.flatMap((r) => r.checks).filter((c) => !c.success); + const checks = results.flatMap((r) => r.checks); + const failed = checks.filter((c) => !c.success); if (failed.length > 0) { return { decision: 'BLOCK', reason: `${failed.length} contract interaction(s) failed verification`, }; } + // No interactions were verified at all (broker returned no contracts, wrong provider name, or a + // tag that matched nothing). "0 verified" is a broker/config gap, not "all verified". + if (checks.length === 0) { + return { + decision: 'WARN', + reason: + 'no consumer contract interactions were verified (no contracts found — broker/tag misconfiguration?)', + }; + } return { decision: 'PASS', reason: 'all contract interactions verified' }; } diff --git a/packages/runner/src/component/component.test.ts b/packages/runner/src/component/component.test.ts index 8241030..439fd44 100644 --- a/packages/runner/src/component/component.test.ts +++ b/packages/runner/src/component/component.test.ts @@ -73,7 +73,9 @@ describe('evaluateComponentGate', () => { expect(gate.decision).toBe('PASS'); }); - it('PASSes on an empty report', () => { - expect(evaluateComponentGate(componentResultsToCtrf([])).decision).toBe('PASS'); + it('WARNs on an empty report — 0 tests collected is "did not measure", not "no failures"', () => { + const gate = evaluateComponentGate(componentResultsToCtrf([])); + expect(gate.decision).toBe('WARN'); + expect(gate.reason).toMatch(/0 tests/i); }); }); diff --git a/packages/runner/src/component/component.ts b/packages/runner/src/component/component.ts index b4e36bb..42459ba 100644 --- a/packages/runner/src/component/component.ts +++ b/packages/runner/src/component/component.ts @@ -67,8 +67,8 @@ export function componentResultsToCtrf(results: ComponentTestResult[]): CTRFRepo /** * Pure gate mapping over the CTRF output of {@link componentResultsToCtrf}: any failed component - * test → `BLOCK`, otherwise `PASS`. Component tests have no "acceptable degradation" tier the way - * a11y/perf budgets do, so there is no `WARN` outcome here. + * test → `BLOCK`; a run that collected zero tests (nothing matched/discovered) → `WARN` (it did + * not measure); otherwise `PASS`. */ export function evaluateComponentGate(report: CTRFReport): GateDecision { const failed = report.results.tests.filter((t) => t.status === 'failed'); @@ -78,6 +78,14 @@ export function evaluateComponentGate(report: CTRFReport): GateDecision { reason: `${failed.length} component test failure(s)`, }; } + // The runner ran but collected no tests (grep/testNamePattern matched nothing, or no + // component tests/stories were discovered). "0 tests" is "did not measure", not "no failures". + if (report.results.tests.length === 0) { + return { + decision: 'WARN', + reason: 'component tier ran but collected 0 tests (nothing matched or was discovered)', + }; + } return { decision: 'PASS', reason: 'no component test failures' }; } diff --git a/packages/runner/src/i18n/i18n.test.ts b/packages/runner/src/i18n/i18n.test.ts index 96f138f..fa26827 100644 --- a/packages/runner/src/i18n/i18n.test.ts +++ b/packages/runner/src/i18n/i18n.test.ts @@ -123,4 +123,16 @@ describe('evaluateI18nGate', () => { expect(evaluateI18nGate(emptyReport, { gate: 'warn' }).decision).toBe('PASS'); expect(evaluateI18nGate(emptyReport, { gate: 'block' }).decision).toBe('PASS'); }); + + it('WARNs when the check compared no locales (measurement gap, not a clean bill)', () => { + const gate = evaluateI18nGate(emptyReport, { gate: 'warn' }, { comparedLocaleCount: 0 }); + expect(gate.decision).toBe('WARN'); + expect(gate.reason).toMatch(/no locales|nothing was measured/i); + }); + + it('PASSes when locales were compared and nothing is missing', () => { + expect( + evaluateI18nGate(emptyReport, { gate: 'warn' }, { comparedLocaleCount: 2 }).decision, + ).toBe('PASS'); + }); }); diff --git a/packages/runner/src/i18n/i18n.ts b/packages/runner/src/i18n/i18n.ts index ed1ad29..e36ffcb 100644 --- a/packages/runner/src/i18n/i18n.ts +++ b/packages/runner/src/i18n/i18n.ts @@ -101,12 +101,29 @@ export interface I18nGateConfig { * Pure gate mapping over the CTRF output of {@link i18nResultsToCtrf}. i18n gaps rarely warrant * blocking a merge, so any missing translation → `WARN` by default; set `cfg.gate` to `'block'` * to treat gaps as blocking, or `'off'` to keep the check informational-only (always `PASS`). + * + * `findMissingTranslations` returns `[]` both when everything is translated AND when it could + * compare nothing (the default locale wasn't loaded, or there are no other locales) — the latter + * is a measurement gap, not a clean bill of health. Pass `measurement.comparedLocaleCount` so the + * gate can `WARN` on that case instead of a false `PASS`. */ -export function evaluateI18nGate(report: CTRFReport, cfg: I18nGateConfig): GateDecision { +export function evaluateI18nGate( + report: CTRFReport, + cfg: I18nGateConfig, + measurement?: { comparedLocaleCount: number }, +): GateDecision { if (cfg.gate === 'off') { return { decision: 'PASS', reason: 'i18n gate is disabled (gate: "off")' }; } + if (measurement && measurement.comparedLocaleCount === 0) { + return { + decision: 'WARN', + reason: + 'i18n check compared no locales (default locale missing, or no other locales to compare) — nothing was measured', + }; + } + const failed = report.results.tests.filter((t) => t.status === 'failed').length; if (failed === 0) { return { decision: 'PASS', reason: 'no missing translations found' }; diff --git a/packages/runner/src/load/k6-load.test.ts b/packages/runner/src/load/k6-load.test.ts index 22ad262..4d7088a 100644 --- a/packages/runner/src/load/k6-load.test.ts +++ b/packages/runner/src/load/k6-load.test.ts @@ -68,31 +68,37 @@ describe('k6LoadResultsToCtrf', () => { describe('evaluateLoadGate', () => { it('PASSes when the summary is within budget', () => { const report = k6LoadResultsToCtrf(withinBudget, thresholds); - expect(evaluateLoadGate(report)).toEqual({ + expect(evaluateLoadGate(report, withinBudget)).toEqual({ decision: 'PASS', reason: expect.any(String), }); }); it('BLOCKs when p95 latency is breached', () => { - const report = k6LoadResultsToCtrf({ ...withinBudget, p95Ms: 900 }, thresholds); - const gate = evaluateLoadGate(report); + const summary = { ...withinBudget, p95Ms: 900 }; + const gate = evaluateLoadGate(k6LoadResultsToCtrf(summary, thresholds), summary); expect(gate.decision).toBe('BLOCK'); expect(gate.reason).toContain('p95Ms'); }); it('BLOCKs when the error rate is breached', () => { - const report = k6LoadResultsToCtrf({ ...withinBudget, errorRate: 0.05 }, thresholds); - const gate = evaluateLoadGate(report); + const summary = { ...withinBudget, errorRate: 0.05 }; + const gate = evaluateLoadGate(k6LoadResultsToCtrf(summary, thresholds), summary); expect(gate.decision).toBe('BLOCK'); expect(gate.reason).toContain('errorRate'); }); it('BLOCKs and reports every breached threshold when multiple are breached', () => { - const report = k6LoadResultsToCtrf(breached, thresholds); - const gate = evaluateLoadGate(report); + const gate = evaluateLoadGate(k6LoadResultsToCtrf(breached, thresholds), breached); expect(gate.decision).toBe('BLOCK'); expect(gate.reason).toContain('p95Ms'); expect(gate.reason).toContain('errorRate'); }); + + it('WARNs when the load run issued zero requests (measured nothing)', () => { + const zeroRequests: K6LoadSummary = { p95Ms: 0, p99Ms: 0, errorRate: 0, requests: 0 }; + const gate = evaluateLoadGate(k6LoadResultsToCtrf(zeroRequests, thresholds), zeroRequests); + expect(gate.decision).toBe('WARN'); + expect(gate.reason).toMatch(/zero requests/i); + }); }); diff --git a/packages/runner/src/load/k6-load.ts b/packages/runner/src/load/k6-load.ts index 965e791..ec37273 100644 --- a/packages/runner/src/load/k6-load.ts +++ b/packages/runner/src/load/k6-load.ts @@ -95,10 +95,19 @@ export function k6LoadResultsToCtrf( /** * Pure gate mapping over the CTRF output of {@link k6LoadResultsToCtrf}: any breached threshold - * (p95, p99, or error rate) → `BLOCK`, otherwise `PASS`. Load thresholds have no "acceptable - * degradation" tier the way a11y/perf budgets do, so there is no `WARN` outcome here. + * (p95, p99, or error rate) → `BLOCK`. A run that issued zero requests measured nothing (missing + * latency/error metrics coerce to 0 and spuriously pass every threshold) → `WARN`. Otherwise `PASS`. */ -export function evaluateLoadGate(report: CTRFReport): GateDecision { +export function evaluateLoadGate(report: CTRFReport, summary: K6LoadSummary): GateDecision { + // Zero requests issued (script error, target unreachable) → the latency/error thresholds were + // never actually measured; they only "passed" because absent metrics normalize to 0. + if (summary.requests === 0) { + return { + decision: 'WARN', + reason: + 'load test issued zero requests — latency and error-rate thresholds were not measured', + }; + } const failed = report.results.tests.filter((t) => t.status === 'failed'); if (failed.length > 0) { return { @@ -197,6 +206,6 @@ export function runK6Load( requests: raw.metrics?.http_reqs?.values?.count ?? 0, }; const report = k6LoadResultsToCtrf(summary, cfg.thresholds); - return { summary, report, gate: evaluateLoadGate(report) }; + return { summary, report, gate: evaluateLoadGate(report, summary) }; }); } diff --git a/packages/runner/src/perf/lighthouse.test.ts b/packages/runner/src/perf/lighthouse.test.ts index 3efafb8..84ed4c2 100644 --- a/packages/runner/src/perf/lighthouse.test.ts +++ b/packages/runner/src/perf/lighthouse.test.ts @@ -143,4 +143,21 @@ describe('evaluatePerfBudgetGate', () => { const gate = evaluatePerfBudgetGate(lighthouseResultsToCtrf(results, budgets)); expect(gate.decision).toBe('PASS'); }); + + it('WARNs when routes were audited but no metrics could be measured', () => { + // A runtime-error report: no categories, no audits → every metric undefined → 0 CTRF tests. + const results: LighthouseRouteResult[] = [ + { route: 'https://preview.example.com/checkout', report: {} }, + ]; + const report = lighthouseResultsToCtrf(results, budgets); + expect(report.results.tests).toHaveLength(0); // pins the "measured nothing" signal + const gate = evaluatePerfBudgetGate(report, results.length); + expect(gate.decision).toBe('WARN'); + expect(gate.reason).toMatch(/no performance metrics/i); + }); + + it('PASSes when no routes were audited at all (nothing to measure)', () => { + const report = lighthouseResultsToCtrf([], budgets); + expect(evaluatePerfBudgetGate(report, 0).decision).toBe('PASS'); + }); }); diff --git a/packages/runner/src/perf/lighthouse.ts b/packages/runner/src/perf/lighthouse.ts index bad3cab..191fc87 100644 --- a/packages/runner/src/perf/lighthouse.ts +++ b/packages/runner/src/perf/lighthouse.ts @@ -162,10 +162,22 @@ export function lighthouseResultsToCtrf( /** * Pure gate mapping over the CTRF output of {@link lighthouseResultsToCtrf}: any `failed` metric - * → `BLOCK`, any `near-budget` tag with no failures → `WARN`, else `PASS`. + * → `BLOCK`, any `near-budget` tag with no failures → `WARN`, else `PASS`. When `auditedRouteCount` + * routes were audited but no metric could be read (a runtime-error report), it is `WARN` — a + * measurement gap must not read as "all budgets satisfied". */ -export function evaluatePerfBudgetGate(report: CTRFReport): GateDecision { +export function evaluatePerfBudgetGate(report: CTRFReport, auditedRouteCount = 0): GateDecision { const { tests } = report.results; + + // Routes were audited but zero metrics were measurable (every Lighthouse metric was undefined, + // e.g. the page failed to load). "Measured nothing" is not "all budgets satisfied". + if (auditedRouteCount > 0 && tests.length === 0) { + return { + decision: 'WARN', + reason: `${auditedRouteCount} route(s) audited but no performance metrics were measured`, + }; + } + const failed = tests.filter((t) => t.status === 'failed'); if (failed.length > 0) { return { @@ -268,5 +280,5 @@ export async function runLighthouseAudit( } const report = lighthouseResultsToCtrf(results, budgets); - return { results, report, gate: evaluatePerfBudgetGate(report) }; + return { results, report, gate: evaluatePerfBudgetGate(report, results.length) }; }