From 497e7b74cfc204d5969b7eed8ef6c12d9761b982 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 10:24:14 +0000 Subject: [PATCH 01/10] docs: add product roadmap and competitive positioning Define the strategic wedge (accurate OpenAPI from real traffic, generated on your own infrastructure) and the four pillars that defend it, in priority order: privacy/local-first, docs-vs-reality drift, measurable accuracy, and interoperable upstream. Includes explicit anti-goals and rough phasing. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012hKk3suH6Hfjqkph6aPVt6 --- ROADMAP.md | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 ROADMAP.md diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..b3f6c3c --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,125 @@ +# EasyDocs Roadmap + +> This is a direction document, not a backlog. It states the wedge we are +> betting on, the pillars that defend it (in priority order), and — just as +> importantly — what we will deliberately **not** build. Tactical issues should +> trace back to a pillar here; anything that doesn't, we say no to. + +## North Star + +**The OpenAPI spec you never have to write, and never have to trust a cloud with.** + +EasyDocs generates accurate, always-current OpenAPI from real traffic, entirely +on your own infrastructure. We own the *source-of-truth generation* layer — the +accurate spec — and let everyone else own presentation and codegen. + +## The gap we are filling + +Look at the field and almost every competitor lands in one of two camps, and +**nobody covers the intersection**: + +- **Spec consumers** — Speakeasy, Fern, ReadMe, Mintlify, Scalar, Stoplight. + They assume you already have a correct OpenAPI file. They don't generate it. + Their blind spot: *the input spec is usually stale or missing.* +- **Traffic-based SaaS** — Postman (Akita), Treblle, Optic Cloud. They capture + traffic, but **your traffic leaves your machine**. Their blind spot: + *anyone who cannot ship production traffic to someone else's cloud* — health, + fintech, public sector, on-prem, air-gapped. + +The empty intersection is: **generate the spec from real traffic, accurately, +without the data ever leaving your infrastructure.** That is exactly what +EasyDocs already is. The strategy is not to invent a new direction — it is to +*refuse to dilute this one* by chasing features from either camp. + +## Pillars, in priority order + +The order is the strategy. When two efforts compete for the same week, the +higher pillar wins. + +### 1. Privacy / local-first — the defensible base (C) + +The one thing SaaS competitors **cannot copy without cannibalizing their own +business model.** Postman and Treblle will not stop being cloud. This is our +terrain, and it is what unlocks the regulated segment (health, fintech, gov, +on-prem) that is structurally locked out of the alternatives. + +Today: PII/secret detection + redaction before anything reaches a hosted +provider; fully offline path via Ollama; deterministic, offline detection. + +Direction: make "nothing leaves the machine" a *provable*, first-class product +promise — not a bullet point. Audit trails of what was redacted, stronger +guarantees and defaults, and positioning aimed squarely at teams that cannot use +a traffic SaaS at all. + +### 2. Drift: docs-vs-reality — the feature only we can build (A) + +Every competitor diffs **spec-against-spec**. We are the only ones who hold, at +the same time, **the committed spec *and* the observed production traffic.** +That enables something no one else can offer: + +> Your `openapi.json` says X, but your API in production actually does Y. + +This is not "did the spec change?" — it is "is the spec still true?" It is +uniquely enabled by having both halves, and it composes directly with what +already exists (traffic capture + `easydocs diff` + the GitHub Action). + +Direction: turn the existing traffic capture and diff machinery into a +drift detector — compare committed spec vs. observed reality and surface where +documentation has diverged, in the dashboard and in CI as an informational +signal. + +### 3. Measurable accuracy — the moat that makes 1 and 2 credible (B) + +Replacing a hand-written spec, and claiming "your docs are wrong," both require +*trust*. Trust comes from being **demonstrably** the most accurate generator of +OpenAPI from traffic. We already have the scaffolding: `apps/evals` and the +cross-provider accuracy `gate`. + +Direction: double down on the eval harness as a moat, not a test. Expand +fixtures and the accuracy scoreboard, keep the cross-provider gate honest, and +make accuracy a number we can publish and defend. + +### 4. Interoperable upstream — the distribution (D) + +We are the source of truth; others are the presentation. Do not build a walled +garden — feed the ecosystem. EasyDocs should produce the clean OpenAPI that +Scalar / ReadMe / Speakeasy / Fern consume. + +Direction: first-class, frictionless export and integrations *into* those tools, +so adopting EasyDocs upstream is the obvious choice regardless of which docs +portal or SDK generator a team already uses. + +## What we will deliberately NOT do + +Saying no here is what keeps the wedge sharp. + +- **No hosted docs portal.** Cede that to ReadMe / Mintlify / Scalar — integrate + with them (pillar 4) instead of competing. +- **No SDK / client generation.** That is Speakeasy / Fern territory. Be their + input, not their rival. +- **No build-blocking CI gatekeeping** for breaking changes. Crowded (Optic, + Bump.sh). Our diff stays informational; our differentiated CI signal is + *drift* (pillar 2), not gating. + +## Phasing + +Rough sequencing, anchored to the pillars above rather than to dates. + +- **Now** — Harden pillar 1 (privacy as a provable promise) and pillar 3 (grow + the eval harness / accuracy scoreboard). These are the foundation the rest + stands on. +- **Next** — Ship pillar 2 (docs-vs-reality drift) on top of the existing + capture + diff + Action, since accuracy (3) is what makes its signal + trustworthy. +- **Later** — Broaden pillar 4 (export/integrations into the docs & SDK + ecosystem) and extend framework/language coverage where it serves the wedge. + +## How we measure the direction is working + +- Regulated / on-prem teams adopting *because* traffic never leaves — a segment + the SaaS camp cannot serve. +- Drift detection surfacing real doc-vs-reality divergences that spec-to-spec + tools structurally cannot see. +- A published, defensible cross-provider accuracy number. +- EasyDocs specs feeding downstream tools (Scalar/ReadMe/Speakeasy/Fern) as the + common upstream. From 7cf044d87b3b49ccbd54b40707e387455a10e567 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 10:30:49 +0000 Subject: [PATCH 02/10] docs: clarify the local dashboard's place in the roadmap Distinguish the producer-side local dashboard (kept, central to pillars 1-2) from a hosted consumer-facing docs portal (anti-goal). Add a section on where the dashboard fits and sharpen the anti-goal wording to remove the apparent contradiction. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012hKk3suH6Hfjqkph6aPVt6 --- ROADMAP.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index b3f6c3c..8ad4a51 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -89,12 +89,27 @@ Direction: first-class, frictionless export and integrations *into* those tools, so adopting EasyDocs upstream is the obvious choice regardless of which docs portal or SDK generator a team already uses. +## Where the dashboard fits + +The local dashboard is the **producer-side cockpit for the source-of-truth +layer**, not a consumer-facing docs site. It runs on your machine, reads from +your database, and is where you review, edit, and approve generated specs, see +version history and field-level diffs, and inspect what was flagged as +sensitive. It is the surface where the pillars become visible: redaction badges +make pillar 1's "nothing leaves the machine" promise tangible, and docs-vs-reality +drift (pillar 2) needs exactly this kind of surface to render. Keeping it local +and producer-facing reinforces the wedge; letting it drift toward a hosted, +consumer-facing portal would cross the anti-goal below. + ## What we will deliberately NOT do Saying no here is what keeps the wedge sharp. -- **No hosted docs portal.** Cede that to ReadMe / Mintlify / Scalar — integrate - with them (pillar 4) instead of competing. +- **No hosted, consumer-facing docs portal.** No multi-tenant hosting, custom + domains, theming, or a published "try it out" aimed at your API's external + consumers. Cede that to ReadMe / Mintlify / Scalar — integrate with them + (pillar 4) instead of competing. This is **not** the local dashboard (see + below), which stays and is central to the wedge. - **No SDK / client generation.** That is Speakeasy / Fern territory. Be their input, not their rival. - **No build-blocking CI gatekeeping** for breaking changes. Crowded (Optic, From 140cdddbd1bd13e263cb6cc6ca2f88ce5ee8d18d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 10:38:12 +0000 Subject: [PATCH 03/10] docs: record the dashboard boundary in AGENTS.md Point contributors to ROADMAP.md and codify the producer-side vs consumer-facing dashboard boundary so future dashboard work respects the strategic line. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012hKk3suH6Hfjqkph6aPVt6 --- AGENTS.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index e34450a..d7aab31 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,16 @@ # EasyDocs +## Product direction + +See `ROADMAP.md` for the wedge and pillars. One boundary matters when touching +the dashboard: it is the **local, producer-side cockpit** for the generated +spec (review/edit/approve, version history + diff, sensitive-field badges, +docs-vs-reality drift). It is deliberately **not** a hosted, consumer-facing +docs portal — no multi-tenant hosting, custom domains, theming, or a published +"try it out" aimed at your API's external consumers (that is ceded to +ReadMe/Mintlify/Scalar). Keep dashboard work serving the developer producing +the spec, not the external API consumer. + ## Release Run `pnpm release` (or `release:minor` / `release:major`). The script From 48b50e3e94108c4ba5698a02698a60dbd5b0c9b9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 10:52:15 +0000 Subject: [PATCH 04/10] feat(core,cli): docs-vs-reality drift detection Add the flagship drift check: compare a committed OpenAPI spec against the spec EasyDocs derives from real traffic, and classify the divergence as undocumented (observed but not in spec), unobserved (documented but not seen), or mismatch (spec contradicts reality). This is the one comparison only EasyDocs can make, since it holds both the committed spec and live traffic. - core: new spec/drift engine (computeDrift/renderDrift/isEmptyDrift/driftCount) built on the existing diffSpecs, scoped to the endpoint contract by default so metadata differences do not create false drift. Exported from the package root and the @easydocs/core/spec/drift subpath, with tests. - cli: new 'easydocs drift [observed]' command. One arg reads observed traffic from the local capture DB; two args compare files. Supports --project and --markdown; informational only, always exits 0. - docs: README, CLI README, and CHANGELOG. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012hKk3suH6Hfjqkph6aPVt6 --- CHANGELOG.md | 13 ++ README.md | 21 ++++ packages/cli/README.md | 29 ++++- packages/cli/src/index.ts | 39 +++++- packages/core/package.json | 5 + packages/core/src/__tests__/drift.test.ts | 91 ++++++++++++++ packages/core/src/index.ts | 8 ++ packages/core/src/spec/drift.ts | 147 ++++++++++++++++++++++ packages/core/tsup.config.ts | 1 + 9 files changed, 351 insertions(+), 3 deletions(-) create mode 100644 packages/core/src/__tests__/drift.test.ts create mode 100644 packages/core/src/spec/drift.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d5b633..44a2daf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- Docs-vs-reality drift detection: a new `easydocs drift ` command compares + a committed OpenAPI spec against the spec EasyDocs derives from real traffic and + reports where documentation has diverged from reality — endpoints/fields observed + but undocumented, documented but never observed, and values that contradict what + traffic shows. With one argument it reads observed traffic from the local capture + DB; with two it compares spec files directly. Takes `--project` and `--markdown` + (for PR comments), and, like `diff`, is informational and never fails the build. + The engine is exported from `@easydocs/core` (`computeDrift` / `renderDrift`, also + at the `@easydocs/core/spec/drift` subpath). + ## [0.7.0] - 2026-07-01 ### Added diff --git a/README.md b/README.md index 1569177..49f69eb 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,27 @@ The check is informational only — it comments the diff, it never fails the bui --- +## Catch drift between your docs and reality + +`diff` compares two spec *files*. **`drift` compares your committed spec against +what your API actually does** — the spec EasyDocs derives from real traffic. It +answers "is my spec still true?", not "did my spec change?". + +```bash +npx easydocs drift openapi.json # against locally captured traffic +npx easydocs drift openapi.json --markdown # PR-comment Markdown +npx easydocs drift committed.json live.json # or compare two files directly +``` + +It surfaces three kinds of divergence: endpoints and fields **observed in traffic +but missing from your spec** (your docs are stale), things **documented but never +observed** (dead or un-exercised), and values where your spec **contradicts** +reality. This is the one check only EasyDocs can run — it's the only tool holding +both the committed spec and the live traffic at the same time. Like `diff`, it's +informational and never fails the build. + +--- + ## How it works 1. Middleware (or proxy) intercepts every request and response diff --git a/packages/cli/README.md b/packages/cli/README.md index 7ec1eae..a5ea9be 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -42,14 +42,41 @@ npx @easydocs/cli export --project=my-api npx @easydocs/cli export > openapi.json ``` +## Check drift — is your committed spec still true? + +`drift` compares a committed spec against reality — the spec EasyDocs derives +from observed traffic. It answers "is my spec still true?", not "did my spec +change?". Only EasyDocs can do this, because it holds both the committed spec and +the live traffic at once. + +```bash +# Compare a committed spec against traffic captured locally +npx @easydocs/cli drift openapi.json + +# Scope to a project +npx @easydocs/cli drift openapi.json --project=my-api + +# Markdown output (for PR comments) +npx @easydocs/cli drift openapi.json --markdown + +# Compare two spec files directly (no DB needed) +npx @easydocs/cli drift committed.json observed.json +``` + +It reports three kinds of divergence — endpoints/fields **observed in traffic +but undocumented**, **documented but never observed**, and values where the docs +**contradict** what traffic shows. Like `diff`, it's informational and always +exits 0. + ## Flags | Flag | Command | Default | Description | |------|---------|---------|-------------| | `--port=` | proxy | `3999` | Port for the proxy server | | `--port=` | dashboard | `4999` | Port for the dashboard | -| `--project=` | proxy, export | `default` | Scope to a project | +| `--project=` | proxy, export, drift | `default` | Scope to a project | | `--yaml` | export | — | Output YAML instead of JSON | +| `--markdown` | diff, drift | — | Emit Markdown (for PR comments) | | `--prod` | dashboard | — | Run `next start` instead of `next dev` | ## Environment variables diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 168c3f7..db6a7ea 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,5 +1,5 @@ import { createDB, getAllEndpoints, getEndpointsByProject, findOrCreateProject, buildFullSpec } from '@easydocs/core' -import { createCapturer, parseConfig, diffSpecs, renderDiff } from '@easydocs/core' +import { createCapturer, parseConfig, diffSpecs, renderDiff, computeDrift, renderDrift } from '@easydocs/core' import type { HttpMethod } from '@easydocs/core' import { createServer } from 'http' import { createRequire } from 'module' @@ -20,13 +20,16 @@ switch (command) { case 'diff': runDiff(args) break + case 'drift': + await runDrift(args) + break case 'proxy': case undefined: await runProxy(args) break default: console.error( - `Unknown command: ${command}\n\nUsage:\n easydocs [proxy] Start proxy server\n easydocs dashboard Start the docs dashboard\n easydocs export Export spec to stdout\n easydocs diff Diff two spec files (JSON or YAML)\n\nFlags:\n --project= Scope to a project (default: all)\n --port= Port for proxy (default: 3999) or dashboard (default: 4999)\n --yaml Export as YAML instead of JSON\n --markdown Diff: emit Markdown (for PR comments) instead of plain text\n --prod Dashboard: run next start instead of next dev` + `Unknown command: ${command}\n\nUsage:\n easydocs [proxy] Start proxy server\n easydocs dashboard Start the docs dashboard\n easydocs export Export spec to stdout\n easydocs diff Diff two spec files (JSON or YAML)\n easydocs drift Check a committed spec against observed traffic\n\nFlags:\n --project= Scope to a project (default: all)\n --port= Port for proxy (default: 3999) or dashboard (default: 4999)\n --yaml Export as YAML instead of JSON\n --markdown Diff/drift: emit Markdown (for PR comments) instead of plain text\n --prod Dashboard: run next start instead of next dev` ) process.exit(1) } @@ -154,6 +157,38 @@ function runDiff(args: string[]) { process.stdout.write(renderDiff(diffSpecs(before, after), { markdown }) + '\n') } +// ─── Drift ──────────────────────────────────────────────────────────────────── + +// Compare a committed spec against reality: the spec EasyDocs derives from +// observed traffic. With one argument, "reality" is read from the local capture +// DB (like export). With two, both sides come from files (handy for tests/CI). +async function runDrift(args: string[]) { + const positionals = args.filter((a) => !a.startsWith('--')) + const [documentedPath, observedPath] = positionals + if (!documentedPath) { + console.error('Usage: easydocs drift [observed-spec] [--project=] [--markdown]') + process.exit(2) + } + + const documented = loadSpecFile(documentedPath) + + let observed: unknown + if (observedPath) { + observed = loadSpecFile(observedPath) + } else { + const projectSlug = getFlag(args, 'project') + const db = createDB(process.env.EASYDOCS_DB_URL) + const endpoints = projectSlug + ? await getEndpointsByProject(db, await findOrCreateProject(db, projectSlug)) + : await getAllEndpoints(db) + observed = buildFullSpec(endpoints, projectSlug ?? undefined) + } + + const markdown = args.includes('--markdown') + // Drift is an informational signal, never a build gate — always exit 0. + process.stdout.write(renderDrift(computeDrift(documented, observed), { markdown }) + '\n') +} + // ─── Proxy ──────────────────────────────────────────────────────────────────── async function runProxy(args: string[]) { diff --git a/packages/core/package.json b/packages/core/package.json index 7790043..aafccb3 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -30,6 +30,11 @@ "import": "./dist/spec/diff.js", "require": "./dist/spec/diff.cjs", "types": "./dist/spec/diff.d.ts" + }, + "./spec/drift": { + "import": "./dist/spec/drift.js", + "require": "./dist/spec/drift.cjs", + "types": "./dist/spec/drift.d.ts" } }, "scripts": { diff --git a/packages/core/src/__tests__/drift.test.ts b/packages/core/src/__tests__/drift.test.ts new file mode 100644 index 0000000..ce81b9a --- /dev/null +++ b/packages/core/src/__tests__/drift.test.ts @@ -0,0 +1,91 @@ +import { describe, it, expect } from 'vitest' +import { computeDrift, renderDrift, isEmptyDrift, driftCount } from '../spec/drift.js' + +// "documented" = the spec you committed; "observed" = what EasyDocs saw in traffic. +const documented = { + info: { title: 'My API', version: '1.0.0' }, + paths: { + '/users': { + get: { + summary: 'List users', + responses: { '200': { description: 'OK' } }, + }, + }, + }, +} + +const observed = { + info: { title: 'API Documentation', version: '1.0.0' }, // metadata differs — must be ignored + paths: { + '/users': { + get: { + summary: 'List users', + responses: { '200': { description: 'Success' } }, // contradicts the docs + }, + }, + '/orders': { + post: { summary: 'Create an order' }, // seen in traffic, undocumented + }, + }, +} + +describe('computeDrift', () => { + it('flags endpoints observed in traffic but missing from the spec', () => { + const report = computeDrift(documented, observed) + expect(report.undocumented.some((f) => f.path.includes('/orders'))).toBe(true) + }) + + it('flags fields documented but contradicted by traffic', () => { + const report = computeDrift(documented, observed) + const mismatch = report.mismatch.find((f) => f.path.endsWith('200.description')) + expect(mismatch?.documented).toBe('OK') + expect(mismatch?.observed).toBe('Success') + }) + + it('flags endpoints documented but never observed', () => { + // Drop /orders from the observed side: now it is documented-but-unobserved. + const report = computeDrift(observed, documented) + expect(report.unobserved.some((f) => f.path.includes('/orders'))).toBe(true) + }) + + it('ignores metadata differences outside the endpoint contract by default', () => { + const report = computeDrift(documented, observed) + const all = [...report.undocumented, ...report.unobserved, ...report.mismatch] + expect(all.every((f) => f.path.startsWith('paths.'))).toBe(true) + expect(all.some((f) => f.path.includes('info.title'))).toBe(false) + }) + + it('honors a custom scope of "" to compare the whole document', () => { + const report = computeDrift(documented, observed, { scope: '' }) + expect(report.mismatch.some((f) => f.path.includes('info.title'))).toBe(true) + }) + + it('reports no drift for a spec that matches its traffic', () => { + const report = computeDrift(documented, documented) + expect(isEmptyDrift(report)).toBe(true) + expect(driftCount(report)).toBe(0) + }) +}) + +describe('renderDrift', () => { + it('renders a no-drift message in both modes', () => { + const report = computeDrift(documented, documented) + expect(renderDrift(report)).toContain('No API drift') + expect(renderDrift(report, { markdown: true })).toContain('No API drift') + }) + + it('renders plain-text sections with a total count', () => { + const txt = renderDrift(computeDrift(documented, observed)) + expect(txt).toMatch(/API drift: \d+ finding/) + expect(txt).toContain('Undocumented') + expect(txt).toContain('Mismatch') + expect(txt).not.toContain('**') + }) + + it('renders markdown sections for PR comments', () => { + const md = renderDrift(computeDrift(documented, observed), { markdown: true }) + expect(md).toContain('**API drift:') + expect(md).toContain('**Undocumented') + expect(md).toContain('→') + }) +}) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5f4a1fd..0246865 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -25,6 +25,14 @@ export { buildOperation } from './spec/builder.js' export { buildFullSpec } from './spec/assemble.js' export { diffSpecs, renderDiff, isEmptyDiff } from './spec/diff.js' export type { SpecDiff, RenderDiffOptions } from './spec/diff.js' +export { computeDrift, renderDrift, isEmptyDrift, driftCount } from './spec/drift.js' +export type { + DriftReport, + DriftFinding, + DriftKind, + ComputeDriftOptions, + RenderDriftOptions, +} from './spec/drift.js' export { OperationSchema } from './spec/schema.js' export type { Operation } from './spec/schema.js' export { buildCaptureEvent, tryParseJson } from './event.js' diff --git a/packages/core/src/spec/drift.ts b/packages/core/src/spec/drift.ts new file mode 100644 index 0000000..bfbb1e1 --- /dev/null +++ b/packages/core/src/spec/drift.ts @@ -0,0 +1,147 @@ +// Docs-vs-reality drift. Unlike `diffSpecs` (which compares two specs +// symmetrically), drift is directional: one side is the spec you *documented* +// (a committed openapi.json), the other is the spec EasyDocs *observed* from +// real traffic. It answers "is my spec still true?", not "did my spec change?". +// +// This is the one comparison only EasyDocs can make, because it is the only tool +// that holds both halves — the committed spec and the live traffic — at once. +// It reuses the field-level `diffSpecs` engine and reinterprets the result: +// +// observed but not documented → undocumented (your spec is stale) +// documented but not observed → unobserved (dead or un-exercised) +// documented value ≠ observed → mismatch (your spec contradicts reality) + +import { diffSpecs } from './diff.js' + +export type DriftKind = 'undocumented' | 'unobserved' | 'mismatch' + +export interface DriftFinding { + path: string + /** Value in the committed spec (present for `unobserved` and `mismatch`). */ + documented?: unknown + /** Value seen in observed traffic (present for `undocumented` and `mismatch`). */ + observed?: unknown +} + +export interface DriftReport { + /** Observed in traffic but missing from the committed spec — the spec is stale. */ + undocumented: DriftFinding[] + /** Present in the committed spec but never observed — possibly dead or un-exercised. */ + unobserved: DriftFinding[] + /** Documented value contradicts what traffic actually shows. */ + mismatch: DriftFinding[] +} + +export interface ComputeDriftOptions { + /** + * Only consider leaf paths under this prefix. Defaults to `'paths.'` so drift + * focuses on the endpoint contract and ignores expected metadata differences + * (info.title, openapi version, components layout) between a hand-written spec + * and a generated one. Pass `''` to compare the whole document. + */ + scope?: string +} + +/** + * Compare a documented spec against an observed one and classify the divergence. + * Pure and deterministic — no I/O. + */ +export function computeDrift( + documented: unknown, + observed: unknown, + opts: ComputeDriftOptions = {} +): DriftReport { + const scope = opts.scope ?? 'paths.' + const inScope = (p: string) => scope === '' || p.startsWith(scope) + + // diffSpecs(documented, observed): `added` is in observed-not-documented, + // `removed` is in documented-not-observed, `changed` is both-but-differ. + const diff = diffSpecs(documented, observed) + + return { + undocumented: diff.added + .filter((f) => inScope(f.path)) + .map((f) => ({ path: f.path, observed: f.value })), + unobserved: diff.removed + .filter((f) => inScope(f.path)) + .map((f) => ({ path: f.path, documented: f.value })), + mismatch: diff.changed + .filter((f) => inScope(f.path)) + .map((f) => ({ path: f.path, documented: f.before, observed: f.after })), + } +} + +export function isEmptyDrift(report: DriftReport): boolean { + return ( + report.undocumented.length === 0 && + report.unobserved.length === 0 && + report.mismatch.length === 0 + ) +} + +export function driftCount(report: DriftReport): number { + return report.undocumented.length + report.unobserved.length + report.mismatch.length +} + +function fmt(value: unknown): string { + return typeof value === 'string' ? value : JSON.stringify(value) +} + +export interface RenderDriftOptions { + markdown?: boolean +} + +/** Render a DriftReport as a human-readable terminal summary, or PR-comment Markdown. */ +export function renderDrift(report: DriftReport, opts: RenderDriftOptions = {}): string { + if (isEmptyDrift(report)) { + return opts.markdown + ? '**No API drift** — your spec matches observed traffic.' + : 'No API drift — your spec matches observed traffic.' + } + + const total = driftCount(report) + const undoc = 'Undocumented — observed in traffic, missing from your spec' + const unobs = 'Documented but unobserved — in your spec, never seen in traffic' + const mism = 'Mismatch — your spec contradicts observed traffic' + + if (opts.markdown) { + const lines: string[] = [ + `**API drift: ${total} finding${total === 1 ? '' : 's'}** — your spec is out of sync with observed traffic.`, + '', + ] + if (report.undocumented.length) { + lines.push(`**${undoc}** (${report.undocumented.length})`, '') + for (const f of report.undocumented) lines.push(`- \`${f.path}\` → ${fmt(f.observed)}`) + lines.push('') + } + if (report.unobserved.length) { + lines.push(`**${unobs}** (${report.unobserved.length})`, '') + for (const f of report.unobserved) lines.push(`- \`${f.path}\` (documented ${fmt(f.documented)})`) + lines.push('') + } + if (report.mismatch.length) { + lines.push(`**${mism}** (${report.mismatch.length})`, '') + for (const f of report.mismatch) lines.push(`- \`${f.path}\`: ${fmt(f.documented)} → ${fmt(f.observed)}`) + lines.push('') + } + return lines.join('\n').trimEnd() + } + + const lines: string[] = [ + `API drift: ${total} finding${total === 1 ? '' : 's'} — your spec is out of sync with observed traffic.`, + '', + ] + if (report.undocumented.length) { + lines.push(`${undoc} (${report.undocumented.length}):`) + for (const f of report.undocumented) lines.push(` + ${f.path} ${fmt(f.observed)}`) + } + if (report.unobserved.length) { + lines.push(`${unobs} (${report.unobserved.length}):`) + for (const f of report.unobserved) lines.push(` - ${f.path} ${fmt(f.documented)}`) + } + if (report.mismatch.length) { + lines.push(`${mism} (${report.mismatch.length}):`) + for (const f of report.mismatch) lines.push(` ~ ${f.path}: ${fmt(f.documented)} -> ${fmt(f.observed)}`) + } + return lines.join('\n').trimEnd() +} diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 9a54f0b..cc868d0 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -6,6 +6,7 @@ export default defineConfig({ 'storage/schema': 'src/storage/schema.ts', 'spec/schema': 'src/spec/schema.ts', 'spec/diff': 'src/spec/diff.ts', + 'spec/drift': 'src/spec/drift.ts', dashboard: 'src/dashboard.ts', }, format: ['esm', 'cjs'], From 3d29cc438a1601815cfd5dcb8216b307f096e10b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 11:33:21 +0000 Subject: [PATCH 05/10] feat(dashboard): docs-vs-reality drift view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface drift in the local producer-side dashboard: read the committed spec from EASYDOCS_SPEC_PATH (default ./openapi.json), compare it against the spec derived from observed traffic, and show where documentation diverged. - lib/db: fetchDrift() loads the committed spec, builds the observed spec, and computes drift plus a per-endpoint finding count for sidebar badges. - new /api/drift route (local only — the comparison never leaves the machine). - Dashboard sidebar: a drift toggle with a total count, and a rose dot on each drifted endpoint alongside the existing conflict dot. - new DriftSummary panel breaking findings into undocumented / mismatch / documented-but-unobserved sections. Keeps the dashboard a producer-side cockpit per the roadmap — no consumer-facing portal, just the surface where the privacy and drift pillars become visible. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012hKk3suH6Hfjqkph6aPVt6 --- CHANGELOG.md | 5 + apps/dashboard/src/app/api/drift/route.ts | 7 ++ apps/dashboard/src/components/Dashboard.tsx | 46 +++++++- .../dashboard/src/components/DriftSummary.tsx | 108 ++++++++++++++++++ apps/dashboard/src/lib/db.ts | 49 ++++++++ 5 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 apps/dashboard/src/app/api/drift/route.ts create mode 100644 apps/dashboard/src/components/DriftSummary.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a2daf..229fd27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (for PR comments), and, like `diff`, is informational and never fails the build. The engine is exported from `@easydocs/core` (`computeDrift` / `renderDrift`, also at the `@easydocs/core/spec/drift` subpath). +- Dashboard drift view: the dashboard now surfaces docs-vs-reality drift. It reads + the committed spec from `EASYDOCS_SPEC_PATH` (default `./openapi.json`), compares + it against observed traffic, badges each drifted endpoint in the sidebar, and + shows a panel breaking down undocumented / mismatch / unobserved findings. Served + by a new local `/api/drift` route — the comparison never leaves the machine. ## [0.7.0] - 2026-07-01 diff --git a/apps/dashboard/src/app/api/drift/route.ts b/apps/dashboard/src/app/api/drift/route.ts new file mode 100644 index 0000000..cf8584f --- /dev/null +++ b/apps/dashboard/src/app/api/drift/route.ts @@ -0,0 +1,7 @@ +import { NextRequest, NextResponse } from 'next/server' +import { fetchDrift } from '@/lib/db' + +export async function GET(req: NextRequest) { + const project = req.nextUrl.searchParams.get('project') ?? undefined + return NextResponse.json(await fetchDrift(project)) +} diff --git a/apps/dashboard/src/components/Dashboard.tsx b/apps/dashboard/src/components/Dashboard.tsx index 61c7376..2834f10 100644 --- a/apps/dashboard/src/components/Dashboard.tsx +++ b/apps/dashboard/src/components/Dashboard.tsx @@ -1,10 +1,12 @@ 'use client' -import { useState, useMemo } from 'react' +import { useState, useMemo, useEffect } from 'react' import { useRouter } from 'next/navigation' import { MethodBadge } from './MethodBadge' import { EndpointDetail } from './EndpointDetail' +import { DriftSummary } from './DriftSummary' import type { Endpoint, Project } from '@easydocs/core/schema' +import type { DriftData } from '@/lib/db' interface Props { endpoints: Endpoint[] @@ -16,6 +18,18 @@ export function Dashboard({ endpoints, projects, currentProject }: Props) { const router = useRouter() const [selected, setSelected] = useState(endpoints[0] ?? null) const [search, setSearch] = useState('') + const [drift, setDrift] = useState(null) + const [showDrift, setShowDrift] = useState(false) + + useEffect(() => { + const q = currentProject ? `?project=${currentProject}` : '' + let active = true + fetch(`/api/drift${q}`) + .then((r) => r.json()) + .then((d: DriftData) => { if (active) setDrift(d) }) + .catch(() => { /* drift is best-effort; ignore fetch errors */ }) + return () => { active = false } + }, [currentProject]) const filtered = useMemo( () => @@ -54,6 +68,26 @@ export function Dashboard({ endpoints, projects, currentProject }: Props) { {endpoints.length} endpoints + {drift?.configured && ( + + )} + {projects.length > 1 && ( - {showDrift && drift?.configured ? ( + {panel === 'drift' && drift?.configured ? ( + ) : panel === 'audit' ? ( + ) : selected ? ( ) : ( diff --git a/apps/dashboard/src/components/RedactionAudit.tsx b/apps/dashboard/src/components/RedactionAudit.tsx new file mode 100644 index 0000000..45dc22c --- /dev/null +++ b/apps/dashboard/src/components/RedactionAudit.tsx @@ -0,0 +1,60 @@ +'use client' + +import { MethodBadge } from './MethodBadge' +import type { Endpoint } from '@easydocs/core/schema' +import type { SensitiveField } from '@easydocs/core/privacy/audit' + +export interface AuditItem { + endpoint: Endpoint + fields: SensitiveField[] +} + +interface Props { + items: AuditItem[] + total: number +} + +export function RedactionAudit({ items, total }: Props) { + const endpointCount = items.length + + return ( +
+
+

Sensitive fields

+

+ {total === 0 ? ( + 'No sensitive fields detected in the captured traffic.' + ) : ( + <> + {total} field{total === 1 ? '' : 's'} across{' '} + {endpointCount} endpoint + {endpointCount === 1 ? '' : 's'} were detected and flagged. Values are redacted + before any payload reaches a hosted AI provider — and kept local when using an + offline model. + + )} +

+
+ + {items.map(({ endpoint, fields }) => ( +
+
+ + {endpoint.path} +
+
    + {fields.map((f, i) => ( +
  • + {f.field} + {f.location} +
    • + ))} +
+
+ ))} +
+ ) +} diff --git a/packages/core/package.json b/packages/core/package.json index aafccb3..ac9284e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -35,6 +35,11 @@ "import": "./dist/spec/drift.js", "require": "./dist/spec/drift.cjs", "types": "./dist/spec/drift.d.ts" + }, + "./privacy/audit": { + "import": "./dist/privacy/audit.js", + "require": "./dist/privacy/audit.cjs", + "types": "./dist/privacy/audit.d.ts" } }, "scripts": { diff --git a/packages/core/src/__tests__/audit.test.ts b/packages/core/src/__tests__/audit.test.ts new file mode 100644 index 0000000..72073b9 --- /dev/null +++ b/packages/core/src/__tests__/audit.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect } from 'vitest' +import { collectSensitiveFields } from '../privacy/audit.js' +import { markSensitiveProperties } from '../privacy/detect.js' +import type { Operation } from '../spec/schema.js' + +function baseOperation(): Operation { + return { + parameters: [ + { name: 'token', in: 'query', required: false }, + { name: 'page', in: 'query', required: false }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + email: { type: 'string' }, + password: { type: 'string' }, + profile: { type: 'object', properties: { ssn: { type: 'string' } } }, + }, + }, + }, + }, + }, + responses: { + '200': { + description: 'OK', + content: { + 'application/json': { + schema: { type: 'object', properties: { id: { type: 'string' }, apiKey: { type: 'string' } } }, + }, + }, + }, + }, + } +} + +describe('collectSensitiveFields', () => { + it('collects fields flagged across parameters, requestBody, and responses', () => { + const op = baseOperation() + markSensitiveProperties(op, new Set(['token', 'password', 'ssn', 'apiKey'])) + + const fields = collectSensitiveFields(op) + const byField = Object.fromEntries(fields.map((f) => [f.field, f.location])) + + expect(byField.token).toBe('query') + expect(byField.password).toBe('requestBody') + expect(byField.ssn).toBe('requestBody') // nested property + expect(byField.apiKey).toBe('response:200') + // Non-sensitive fields are not reported. + expect(fields.some((f) => f.field === 'page')).toBe(false) + expect(fields.some((f) => f.field === 'id')).toBe(false) + }) + + it('returns an empty array when nothing is flagged', () => { + expect(collectSensitiveFields(baseOperation())).toEqual([]) + }) + + it('de-duplicates by location and field', () => { + const op: Operation = { + responses: { + '200': { + description: 'OK', + content: { + 'application/json': { schema: { type: 'object', properties: { token: { type: 'string' } } } }, + 'application/xml': { schema: { type: 'object', properties: { token: { type: 'string' } } } }, + }, + }, + }, + } + markSensitiveProperties(op, new Set(['token'])) + const fields = collectSensitiveFields(op) + expect(fields).toHaveLength(1) + expect(fields[0]).toEqual({ location: 'response:200', field: 'token' }) + }) +}) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d8c3f01..882bf0c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -41,5 +41,7 @@ export { parseConfig, EasyDocsConfigSchema } from './types.js' export type { CaptureEvent, EasyDocsConfig, HttpMethod, AIConfig, StorageConfig, PrivacyConfig } from './types.js' export { detect, markSensitiveProperties } from './privacy/detect.js' export type { DetectResult } from './privacy/detect.js' +export { collectSensitiveFields } from './privacy/audit.js' +export type { SensitiveField } from './privacy/audit.js' export { resolveModel, resolveProvider, isHostedProvider } from './ai/provider.js' export type { Provider } from './ai/provider.js' diff --git a/packages/core/src/privacy/audit.ts b/packages/core/src/privacy/audit.ts new file mode 100644 index 0000000000000000000000000000000000000000..5d11ed4e47cb585fbd8491557cd8c9d6d2c023a1 GIT binary patch literal 2552 zcma)8+iu%N5bZO+VhjWklpCxyqbwYEoUUT|#$t#^j7D3Z2nPSz0pJ)S4z^YdN0~ zZ`kap&|K#9UfJDmO=hVm)zcH_QJNA`EZ0^$y6+Y%$-2gHPPXMt5M%39)#AZ&=L0_t@b{)^**Fv!C-kt=mZ~Q2N+!c9w}5Ia z_t5A=Wov3WHxWqNkdmhIE<@Q2^ei*+7idxLXuDGG*c)IG=@>HAPxSBK{{V6=|AMMu zl})C~X`zjEl+yi&4^yiOmYhnXK3+F!DWnt<46;y7QC2ElCCC?u`_3>l$|>iT>zwOs zH!Tbp#ctO^zNRjBjlmNET$*NU*$CXSt8*0GXyxTx{5%MsW)L=Q zjozqim->k>$ac#X)RHLb;iM)#(;c)`-Wq!<&sOMoGzIk!u5O@rv-}KOj>a&&>r9QDFb<1h^l_m7kumlhnn>%Dfzs_E zTDtZbN#8#fsxKctdi!=L-iMym7OF%A$h$THqnkNJBF2x_6W~j<-$!^gov)`*QP8n+p}}q`|#wxZnaza?Y9^$ zR{pTF6P%K1-nFHM^KqDgi0e6+kbVp;Ch&$B#~)~}S~wpETge5)(XSf;e>z(M0Hwn| zAAFw$uIF2AW7(yC?GC))@AI(vqn~zg&c&60m*)eePzK`eD}TBFq*iP zud8qD$M~gbE?8woZKAQ={Y_{Se>i%}oSNXs<>YL=OVGB%UR>y)awn^^Q6)~@QPAWB zHp{saCTSKa^s literal 0 HcmV?d00001 diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index cc868d0..216ae57 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -7,6 +7,7 @@ export default defineConfig({ 'spec/schema': 'src/spec/schema.ts', 'spec/diff': 'src/spec/diff.ts', 'spec/drift': 'src/spec/drift.ts', + 'privacy/audit': 'src/privacy/audit.ts', dashboard: 'src/dashboard.ts', }, format: ['esm', 'cjs'], From 056aed3d0c22948393083505307c0ea45cd2e881 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 14:54:48 +0000 Subject: [PATCH 09/10] feat(evals): per-section accuracy breakdown in the scoreboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scorer already computes per-dimension scores; the matrix threw them away. Aggregate them so the scoreboard shows where each model is strong or weak (tags, responses, response schema, parameters, request body, security), not just an overall mean — making the accuracy moat granular and diagnosable. - score: new pure meanByDimension() that averages each acc:
across the fixtures where it applied. - matrix: capture namedScores per fixture, render an 'Accuracy by section' table in both the console and --markdown scoreboard. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012hKk3suH6Hfjqkph6aPVt6 --- CHANGELOG.md | 5 ++++- apps/evals/matrix.ts | 38 +++++++++++++++++++++++++++++++------- apps/evals/score.ts | 26 ++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ae487c..7fde2a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Publishable accuracy scoreboard: `pnpm matrix --markdown` emits the per-provider/ model spec-accuracy scoreboard as a committable Markdown report (progress goes to stderr so stdout is clean). Run it where provider keys are available: - `pnpm matrix --markdown > SCOREBOARD.md`. + `pnpm matrix --markdown > SCOREBOARD.md`. The scoreboard now also breaks accuracy + down **by spec section** (tags, responses, response schema, parameters, request + body, security) so it's clear where a model is strong or weak, not just its overall + mean. `meanByDimension` is exported from the eval scorer. - Redaction audit: EasyDocs can now show exactly which fields it protects. A new `collectSensitiveFields` helper (exported from `@easydocs/core` and the pure `@easydocs/core/privacy/audit` subpath) reads the `x-easydocs-sensitive` markers diff --git a/apps/evals/matrix.ts b/apps/evals/matrix.ts index 5cd7ba3..b78189e 100644 --- a/apps/evals/matrix.ts +++ b/apps/evals/matrix.ts @@ -17,7 +17,7 @@ import { buildOperation } from '@easydocs/core' import { readFileSync, readdirSync } from 'fs' import { resolve, join } from 'path' -import score from './score.ts' +import score, { meanByDimension } from './score.ts' const DIR = import.meta.dirname try { @@ -96,15 +96,16 @@ async function runModel(c: Candidate) { try { const spec = await buildOperation(fixture, null, { provider: c.provider, model: c.model }) const r = score(JSON.stringify(spec), { vars: { fixture: rel } }) - return { rel, score: r.score, reason: r.reason } + return { rel, score: r.score, reason: r.reason, named: r.namedScores } } catch (err) { - return { rel, score: 0, reason: `ERROR: ${String(err).slice(0, 120)}` } + return { rel, score: 0, reason: `ERROR: ${String(err).slice(0, 120)}`, named: undefined } } }) ) const mean = results.reduce((s, r) => s + r.score, 0) / results.length const worst = results.reduce((a, b) => (b.score < a.score ? b : a)) - return { mean, worst, results } + const dims = meanByDimension(results.map((r) => r.named)) + return { mean, worst, results, dims } } const label = (c: Candidate) => `${c.provider}/${c.model}` @@ -153,7 +154,7 @@ if (GATE) { process.exit(0) } -const ran: { c: Candidate; mean: number; worst: any }[] = [] +const ran: { c: Candidate; mean: number; worst: any; dims: Record }[] = [] const skipped: string[] = [] for (const c of MATRIX) { @@ -163,8 +164,8 @@ for (const c of MATRIX) { } // In markdown mode stdout is the report, so progress goes to stderr. ;(MARKDOWN ? process.stderr : process.stdout).write(`running ${label(c)} (${fixtures.length} fixtures)...\n`) - const { mean, worst, results } = await runModel(c) - ran.push({ c, mean, worst }) + const { mean, worst, results, dims } = await runModel(c) + ran.push({ c, mean, worst, dims }) if (!QUIET) { for (const r of results.filter((r) => r.score < 1)) { console.log(` ${r.rel.replace('fixtures/', '').padEnd(38)} ${r.score.toFixed(3)} ${r.reason}`) @@ -174,6 +175,14 @@ for (const c of MATRIX) { const sorted = ran.sort((a, b) => b.mean - a.mean) +// The union of section names seen across all models, in a stable display order. +const SECTION_ORDER = ['tags', 'responses', 'responseSchema', 'parameters', 'requestBody', 'security'] +const allDims = [ + ...SECTION_ORDER.filter((d) => sorted.some((r) => d in r.dims)), + ...[...new Set(sorted.flatMap((r) => Object.keys(r.dims)))].filter((d) => !SECTION_ORDER.includes(d)).sort(), +] +const cell = (r: (typeof sorted)[number], d: string) => (d in r.dims ? r.dims[d].toFixed(2) : '—') + if (MARKDOWN) { const lines: string[] = [ '# EasyDocs spec-accuracy scoreboard', @@ -194,6 +203,14 @@ if (MARKDOWN) { if (sorted.length === 0) { lines.push('| _no models available_ | — | — |') } + if (allDims.length > 0) { + lines.push('', '## Accuracy by section', '', 'Where each model is strong or weak. A section only counts on the fixtures where it applies.', '') + lines.push(`| Model | ${allDims.join(' | ')} |`) + lines.push(`| ----- | ${allDims.map(() => '---').join(' | ')} |`) + for (const r of sorted) { + lines.push(`| \`${label(r.c)}\` | ${allDims.map((d) => cell(r, d)).join(' | ')} |`) + } + } if (skipped.length) { lines.push('', '**Skipped (no credentials):**', '') for (const s of skipped) lines.push(`- ${s}`) @@ -206,6 +223,13 @@ if (MARKDOWN) { const w = `${r.worst.rel.replace('fixtures/', '')} (${r.worst.score.toFixed(2)})` console.log(label(r.c).padEnd(40), r.mean.toFixed(3), ' ', w) } + if (allDims.length > 0) { + console.log('\n=== ACCURACY BY SECTION ===') + console.log('model'.padEnd(40), allDims.map((d) => d.slice(0, 8).padStart(8)).join(' ')) + for (const r of sorted) { + console.log(label(r.c).padEnd(40), allDims.map((d) => cell(r, d).padStart(8)).join(' ')) + } + } if (skipped.length) { console.log('\n=== SKIPPED (no credentials) ===') for (const s of skipped) console.log(' -', s) diff --git a/apps/evals/score.ts b/apps/evals/score.ts index ad30eee..03e6d49 100644 --- a/apps/evals/score.ts +++ b/apps/evals/score.ts @@ -18,6 +18,32 @@ type GradingResult = { namedScores?: Record } +/** + * Average each accuracy dimension across a set of per-fixture `namedScores` + * (the `acc:` map score() returns). A dimension only counts on the + * fixtures where it applied, so e.g. `security` isn't dragged down by fixtures + * that have no auth. Returns dimension → mean, sorted for stable output. + */ +export function meanByDimension( + runs: Array | undefined> +): Record { + const acc: Record = {} + for (const run of runs) { + if (!run) continue + for (const [rawKey, value] of Object.entries(run)) { + const key = rawKey.startsWith('acc:') ? rawKey.slice(4) : rawKey + const entry = (acc[key] ??= { sum: 0, count: 0 }) + entry.sum += value + entry.count += 1 + } + } + return Object.fromEntries( + Object.keys(acc) + .sort() + .map((k) => [k, acc[k].sum / acc[k].count]) + ) +} + /** F1 over two string sets. Both empty → 1 (nothing expected, nothing produced). */ function setF1(expected: string[], actual: string[]): number { const e = new Set(expected) From 326217dbad12ef6953369160b4d18fe921294533 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Jul 2026 09:53:28 +0000 Subject: [PATCH 10/10] feat(core,cli): easydocs audit command for the redaction inventory Add a scriptable/CI surface for the redaction audit, mirroring the dashboard's 'Sensitive fields' panel: 'easydocs audit' reads the stored specs and lists every field flagged sensitive, grouped by endpoint. Supports --project and --markdown (for PR comments). Lets a pipeline assert exactly what EasyDocs is protecting. - core: renderAudit() + EndpointAudit type in privacy/audit (pure), exported and tested. - cli: new 'audit' command reading endpoints from the capture DB. - docs: CLI README + CHANGELOG. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_012hKk3suH6Hfjqkph6aPVt6 --- CHANGELOG.md | 2 ++ packages/cli/README.md | 16 ++++++++++-- packages/cli/src/index.ts | 29 ++++++++++++++++++++-- packages/core/src/__tests__/audit.test.ts | 24 +++++++++++++++++- packages/core/src/index.ts | 4 +-- packages/core/src/privacy/audit.ts | Bin 2552 -> 4031 bytes 6 files changed, 68 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fde2a9..0554452 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `@easydocs/core/privacy/audit` subpath) reads the `x-easydocs-sensitive` markers back out of a spec, and the dashboard surfaces them in a "Sensitive fields" panel grouped by endpoint — making the PII-safe promise provable, not just a claim. + A new `easydocs audit` command lists the same inventory from the terminal (with + `--project` and `--markdown`), so a pipeline can assert what is being protected. ## [0.7.0] - 2026-07-01 diff --git a/packages/cli/README.md b/packages/cli/README.md index a5ea9be..2f228b1 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -68,15 +68,27 @@ but undocumented**, **documented but never observed**, and values where the docs **contradict** what traffic shows. Like `diff`, it's informational and always exits 0. +## Audit sensitive fields + +`audit` lists every field EasyDocs flagged as sensitive across the stored specs — +the scriptable/CI counterpart to the dashboard's "Sensitive fields" panel. Useful +to prove, in a pipeline, exactly what is being protected. + +```bash +npx @easydocs/cli audit # all projects +npx @easydocs/cli audit --project=my-api +npx @easydocs/cli audit --markdown # for a PR comment +``` + ## Flags | Flag | Command | Default | Description | |------|---------|---------|-------------| | `--port=` | proxy | `3999` | Port for the proxy server | | `--port=` | dashboard | `4999` | Port for the dashboard | -| `--project=` | proxy, export, drift | `default` | Scope to a project | +| `--project=` | proxy, export, drift, audit | `default` | Scope to a project | | `--yaml` | export | — | Output YAML instead of JSON | -| `--markdown` | diff, drift | — | Emit Markdown (for PR comments) | +| `--markdown` | diff, drift, audit | — | Emit Markdown (for PR comments) | | `--prod` | dashboard | — | Run `next start` instead of `next dev` | ## Environment variables diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index db6a7ea..cb8e7eb 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,6 +1,7 @@ import { createDB, getAllEndpoints, getEndpointsByProject, findOrCreateProject, buildFullSpec } from '@easydocs/core' import { createCapturer, parseConfig, diffSpecs, renderDiff, computeDrift, renderDrift } from '@easydocs/core' -import type { HttpMethod } from '@easydocs/core' +import { collectSensitiveFields, renderAudit } from '@easydocs/core' +import type { HttpMethod, EndpointAudit } from '@easydocs/core' import { createServer } from 'http' import { createRequire } from 'module' import { existsSync, readFileSync } from 'fs' @@ -23,13 +24,16 @@ switch (command) { case 'drift': await runDrift(args) break + case 'audit': + await runAudit(args) + break case 'proxy': case undefined: await runProxy(args) break default: console.error( - `Unknown command: ${command}\n\nUsage:\n easydocs [proxy] Start proxy server\n easydocs dashboard Start the docs dashboard\n easydocs export Export spec to stdout\n easydocs diff Diff two spec files (JSON or YAML)\n easydocs drift Check a committed spec against observed traffic\n\nFlags:\n --project= Scope to a project (default: all)\n --port= Port for proxy (default: 3999) or dashboard (default: 4999)\n --yaml Export as YAML instead of JSON\n --markdown Diff/drift: emit Markdown (for PR comments) instead of plain text\n --prod Dashboard: run next start instead of next dev` + `Unknown command: ${command}\n\nUsage:\n easydocs [proxy] Start proxy server\n easydocs dashboard Start the docs dashboard\n easydocs export Export spec to stdout\n easydocs diff Diff two spec files (JSON or YAML)\n easydocs drift Check a committed spec against observed traffic\n easydocs audit List sensitive fields detected across the specs\n\nFlags:\n --project= Scope to a project (default: all)\n --port= Port for proxy (default: 3999) or dashboard (default: 4999)\n --yaml Export as YAML instead of JSON\n --markdown Diff/drift/audit: emit Markdown (for PR comments) instead of plain text\n --prod Dashboard: run next start instead of next dev` ) process.exit(1) } @@ -189,6 +193,27 @@ async function runDrift(args: string[]) { process.stdout.write(renderDrift(computeDrift(documented, observed), { markdown }) + '\n') } +// ─── Audit ──────────────────────────────────────────────────────────────────── + +// List every field EasyDocs flagged sensitive across the stored specs — the +// scriptable/CI counterpart to the dashboard's "Sensitive fields" panel. +async function runAudit(args: string[]) { + const projectSlug = getFlag(args, 'project') + const db = createDB(process.env.EASYDOCS_DB_URL) + const endpoints = projectSlug + ? await getEndpointsByProject(db, await findOrCreateProject(db, projectSlug)) + : await getAllEndpoints(db) + + const items: EndpointAudit[] = endpoints + .map((e) => { + const spec = e.isManuallyEdited && e.manualSpec ? e.manualSpec : e.spec + return { path: e.path, method: e.method, fields: spec ? collectSensitiveFields(spec) : [] } + }) + .sort((a, b) => a.path.localeCompare(b.path) || a.method.localeCompare(b.method)) + + process.stdout.write(renderAudit(items, { markdown: args.includes('--markdown') }) + '\n') +} + // ─── Proxy ──────────────────────────────────────────────────────────────────── async function runProxy(args: string[]) { diff --git a/packages/core/src/__tests__/audit.test.ts b/packages/core/src/__tests__/audit.test.ts index 72073b9..240d18b 100644 --- a/packages/core/src/__tests__/audit.test.ts +++ b/packages/core/src/__tests__/audit.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from 'vitest' -import { collectSensitiveFields } from '../privacy/audit.js' +import { collectSensitiveFields, renderAudit } from '../privacy/audit.js' +import type { EndpointAudit } from '../privacy/audit.js' import { markSensitiveProperties } from '../privacy/detect.js' import type { Operation } from '../spec/schema.js' @@ -57,6 +58,27 @@ describe('collectSensitiveFields', () => { expect(collectSensitiveFields(baseOperation())).toEqual([]) }) + it('renders a per-endpoint audit in plain text and markdown', () => { + const items: EndpointAudit[] = [ + { path: '/auth/login', method: 'POST', fields: [{ location: 'requestBody', field: 'password' }] }, + { path: '/products', method: 'GET', fields: [] }, // no sensitive fields → omitted + ] + const txt = renderAudit(items) + expect(txt).toMatch(/1 sensitive field across 1 endpoint/) + expect(txt).toContain('POST /auth/login') + expect(txt).toContain('password (requestBody)') + expect(txt).not.toContain('/products') + + const md = renderAudit(items, { markdown: true }) + expect(md).toContain('`POST /auth/login`') + expect(md).toContain('- `password` (requestBody)') + }) + + it('renders a no-fields message in both modes', () => { + expect(renderAudit([])).toContain('No sensitive fields') + expect(renderAudit([], { markdown: true })).toContain('No sensitive fields') + }) + it('de-duplicates by location and field', () => { const op: Operation = { responses: { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 882bf0c..4724fa4 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -41,7 +41,7 @@ export { parseConfig, EasyDocsConfigSchema } from './types.js' export type { CaptureEvent, EasyDocsConfig, HttpMethod, AIConfig, StorageConfig, PrivacyConfig } from './types.js' export { detect, markSensitiveProperties } from './privacy/detect.js' export type { DetectResult } from './privacy/detect.js' -export { collectSensitiveFields } from './privacy/audit.js' -export type { SensitiveField } from './privacy/audit.js' +export { collectSensitiveFields, renderAudit } from './privacy/audit.js' +export type { SensitiveField, EndpointAudit, RenderAuditOptions } from './privacy/audit.js' export { resolveModel, resolveProvider, isHostedProvider } from './ai/provider.js' export type { Provider } from './ai/provider.js' diff --git a/packages/core/src/privacy/audit.ts b/packages/core/src/privacy/audit.ts index 5d11ed4e47cb585fbd8491557cd8c9d6d2c023a1..75c8a498f28b89e0a7da00835130d111cc8afea6 100644 GIT binary patch delta 1498 zcmai!O>WyT5QW$Ewg>221dy!QO1m|n1YNYd{vcf@2FysKh>b)FBAqla1oaR-NN>=) z^b}ckNJ)v3xUF^^aUSP=GvwXNU$1`Nz3682t=5f&RF;}tq=a`Wt2Hm*ZZm1In*p`3 z#R|qYQeE=0B3tNe638W$nOWfrDI=}CruQyzww|@K=klkdGHSf|r`mQ>E8`ETM01tt z8+Ee6h1Mkr<?wYHF68?W;pWnMf_CL-<04;T`Oul1h{?+lmEmp&NWYU8cIK zNLhRs&WI5m^_6uN`BwEh%|m6Ov{bR9M=Z|P3+UQ%_^}6XA;GSlk4(Iwl$}}JNLzRu z1_``soXe8y3`03b(j7x`!10SqQkS;CF<#G^a-VLsWjD!0CT8Q?l)}&~;1rHwaQIYY z;WSu8<_HIgB;lAZ49w1M8|BP&CLY1_cK4kih@y`gW^5+<@{p0G)KbPBkqWjTq@rdW z%3vFj=Q53hL1TsBUka1_?zsSWeRN-t$V#aQi5K1+?7W%wX>mgw37dT590cGJOwdB4 zjWz}cyN5muoLg*gIP09{%{Wwhw(ZgU{r$&13<#ACqsSaCDAx^f?3VA$ZK*}(4{>J% z4T-cMH>8C&j1hICuVu$~0pS(Z7Nt(P)i(txYz^NumVwT5@4wO+m2>9H99&P`7~VH- z>}o$TZaRzc`i4u-qj?R+!pO{xfj zKG^!BhH38qf90nu49-