diff --git a/AGENTS.md b/AGENTS.md index 626e22d..f304b6a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,16 @@ # EasyDocs +## Product direction + +See `ROADMAP.md` for the wedge and pillars. One boundary matters when touching +the dashboard: it is the **local, producer-side cockpit** for the generated +spec (review/edit/approve, version history + diff, sensitive-field badges, +docs-vs-reality drift). It is deliberately **not** a hosted, consumer-facing +docs portal — no multi-tenant hosting, custom domains, theming, or a published +"try it out" aimed at your API's external consumers (that is ceded to +ReadMe/Mintlify/Scalar). Keep dashboard work serving the developer producing +the spec, not the external API consumer. + ## Release Run `pnpm release` (or `release:minor` / `release:major`). The script diff --git a/CHANGELOG.md b/CHANGELOG.md index b159122..3b04cd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,36 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- Docs-vs-reality drift detection: a new `easydocs drift ` command compares + a committed OpenAPI spec against the spec EasyDocs derives from real traffic and + reports where documentation has diverged from reality — endpoints/fields observed + but undocumented, documented but never observed, and values that contradict what + traffic shows. With one argument it reads observed traffic from the local capture + DB; with two it compares spec files directly. Takes `--project` and `--markdown` + (for PR comments), and, like `diff`, is informational and never fails the build. + The engine is exported from `@easydocs/core` (`computeDrift` / `renderDrift`, also + at the `@easydocs/core/spec/drift` subpath). +- Dashboard drift view: the dashboard now surfaces docs-vs-reality drift. It reads + the committed spec from `EASYDOCS_SPEC_PATH` (default `./openapi.json`), compares + it against observed traffic, badges each drifted endpoint in the sidebar, and + shows a panel breaking down undocumented / mismatch / unobserved findings. Served + by a new local `/api/drift` route — the comparison never leaves the machine. +- Strict offline mode (`privacy.offline: true`): a hard local-first guarantee for + regulated / air-gapped setups. EasyDocs pins itself to a local Ollama model, + ignores any hosted API keys present in the environment, and fails fast at startup + if a hosted provider is explicitly configured — so no captured payload can ever + reach a third-party service. `isHostedProvider` is exported from `@easydocs/core`. +- Redaction audit: EasyDocs can now show exactly which fields it protects. A new + `collectSensitiveFields` helper (exported from `@easydocs/core` and the pure + `@easydocs/core/privacy/audit` subpath) reads the `x-easydocs-sensitive` markers + back out of a spec, and the dashboard surfaces them in a "Sensitive fields" panel + grouped by endpoint — making the PII-safe promise provable, not just a claim. + A new `easydocs audit` command lists the same inventory from the terminal (with + `--project` and `--markdown`), so a pipeline can assert what is being protected. + ## [0.8.1] - 2026-07-02 ### Changed diff --git a/README.md b/README.md index 39f57ac..3ccc668 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,27 @@ the PR when the spec changes cross that threshold; the comment is still posted. --- +## Catch drift between your docs and reality + +`diff` compares two spec *files*. **`drift` compares your committed spec against +what your API actually does** — the spec EasyDocs derives from real traffic. It +answers "is my spec still true?", not "did my spec change?". + +```bash +npx easydocs drift openapi.json # against locally captured traffic +npx easydocs drift openapi.json --markdown # PR-comment Markdown +npx easydocs drift committed.json live.json # or compare two files directly +``` + +It surfaces three kinds of divergence: endpoints and fields **observed in traffic +but missing from your spec** (your docs are stale), things **documented but never +observed** (dead or un-exercised), and values where your spec **contradicts** +reality. This is the one check only EasyDocs can run — it's the only tool holding +both the committed spec and the live traffic at the same time. Like `diff`, it's +informational and never fails the build. + +--- + ## How it works 1. Middleware (or proxy) intercepts every request and response @@ -179,6 +200,7 @@ easydocs({ }, privacy: { enabled: true, // on by default; detect & redact PII/secrets + offline: false, // strict local-first: only ever use a local Ollama model placeholder: "[REDACTED]", // value substituted for sensitive fields allowlist: ["public_token"], // key names never to flag customRules: { @@ -198,6 +220,12 @@ to a **hosted** provider (OpenAI/Anthropic/DeepSeek); with local Ollama nothing the machine, so real values are kept for accuracy. Flagged fields are marked in the spec with `x-easydocs-sensitive` and shown with a badge in the dashboard. +For regulated or air-gapped environments, set `privacy.offline: true` for a hard +guarantee: EasyDocs pins itself to a local Ollama model, **ignores any hosted API +keys in the environment**, and refuses to start if a hosted provider is explicitly +configured. Nothing captured can ever reach a third-party service — redaction becomes +moot because no payload leaves the machine at all. + --- ## Multiple projects diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..8ad4a51 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,140 @@ +# EasyDocs Roadmap + +> This is a direction document, not a backlog. It states the wedge we are +> betting on, the pillars that defend it (in priority order), and — just as +> importantly — what we will deliberately **not** build. Tactical issues should +> trace back to a pillar here; anything that doesn't, we say no to. + +## North Star + +**The OpenAPI spec you never have to write, and never have to trust a cloud with.** + +EasyDocs generates accurate, always-current OpenAPI from real traffic, entirely +on your own infrastructure. We own the *source-of-truth generation* layer — the +accurate spec — and let everyone else own presentation and codegen. + +## The gap we are filling + +Look at the field and almost every competitor lands in one of two camps, and +**nobody covers the intersection**: + +- **Spec consumers** — Speakeasy, Fern, ReadMe, Mintlify, Scalar, Stoplight. + They assume you already have a correct OpenAPI file. They don't generate it. + Their blind spot: *the input spec is usually stale or missing.* +- **Traffic-based SaaS** — Postman (Akita), Treblle, Optic Cloud. They capture + traffic, but **your traffic leaves your machine**. Their blind spot: + *anyone who cannot ship production traffic to someone else's cloud* — health, + fintech, public sector, on-prem, air-gapped. + +The empty intersection is: **generate the spec from real traffic, accurately, +without the data ever leaving your infrastructure.** That is exactly what +EasyDocs already is. The strategy is not to invent a new direction — it is to +*refuse to dilute this one* by chasing features from either camp. + +## Pillars, in priority order + +The order is the strategy. When two efforts compete for the same week, the +higher pillar wins. + +### 1. Privacy / local-first — the defensible base (C) + +The one thing SaaS competitors **cannot copy without cannibalizing their own +business model.** Postman and Treblle will not stop being cloud. This is our +terrain, and it is what unlocks the regulated segment (health, fintech, gov, +on-prem) that is structurally locked out of the alternatives. + +Today: PII/secret detection + redaction before anything reaches a hosted +provider; fully offline path via Ollama; deterministic, offline detection. + +Direction: make "nothing leaves the machine" a *provable*, first-class product +promise — not a bullet point. Audit trails of what was redacted, stronger +guarantees and defaults, and positioning aimed squarely at teams that cannot use +a traffic SaaS at all. + +### 2. Drift: docs-vs-reality — the feature only we can build (A) + +Every competitor diffs **spec-against-spec**. We are the only ones who hold, at +the same time, **the committed spec *and* the observed production traffic.** +That enables something no one else can offer: + +> Your `openapi.json` says X, but your API in production actually does Y. + +This is not "did the spec change?" — it is "is the spec still true?" It is +uniquely enabled by having both halves, and it composes directly with what +already exists (traffic capture + `easydocs diff` + the GitHub Action). + +Direction: turn the existing traffic capture and diff machinery into a +drift detector — compare committed spec vs. observed reality and surface where +documentation has diverged, in the dashboard and in CI as an informational +signal. + +### 3. Measurable accuracy — the moat that makes 1 and 2 credible (B) + +Replacing a hand-written spec, and claiming "your docs are wrong," both require +*trust*. Trust comes from being **demonstrably** the most accurate generator of +OpenAPI from traffic. We already have the scaffolding: `apps/evals` and the +cross-provider accuracy `gate`. + +Direction: double down on the eval harness as a moat, not a test. Expand +fixtures and the accuracy scoreboard, keep the cross-provider gate honest, and +make accuracy a number we can publish and defend. + +### 4. Interoperable upstream — the distribution (D) + +We are the source of truth; others are the presentation. Do not build a walled +garden — feed the ecosystem. EasyDocs should produce the clean OpenAPI that +Scalar / ReadMe / Speakeasy / Fern consume. + +Direction: first-class, frictionless export and integrations *into* those tools, +so adopting EasyDocs upstream is the obvious choice regardless of which docs +portal or SDK generator a team already uses. + +## Where the dashboard fits + +The local dashboard is the **producer-side cockpit for the source-of-truth +layer**, not a consumer-facing docs site. It runs on your machine, reads from +your database, and is where you review, edit, and approve generated specs, see +version history and field-level diffs, and inspect what was flagged as +sensitive. It is the surface where the pillars become visible: redaction badges +make pillar 1's "nothing leaves the machine" promise tangible, and docs-vs-reality +drift (pillar 2) needs exactly this kind of surface to render. Keeping it local +and producer-facing reinforces the wedge; letting it drift toward a hosted, +consumer-facing portal would cross the anti-goal below. + +## What we will deliberately NOT do + +Saying no here is what keeps the wedge sharp. + +- **No hosted, consumer-facing docs portal.** No multi-tenant hosting, custom + domains, theming, or a published "try it out" aimed at your API's external + consumers. Cede that to ReadMe / Mintlify / Scalar — integrate with them + (pillar 4) instead of competing. This is **not** the local dashboard (see + below), which stays and is central to the wedge. +- **No SDK / client generation.** That is Speakeasy / Fern territory. Be their + input, not their rival. +- **No build-blocking CI gatekeeping** for breaking changes. Crowded (Optic, + Bump.sh). Our diff stays informational; our differentiated CI signal is + *drift* (pillar 2), not gating. + +## Phasing + +Rough sequencing, anchored to the pillars above rather than to dates. + +- **Now** — Harden pillar 1 (privacy as a provable promise) and pillar 3 (grow + the eval harness / accuracy scoreboard). These are the foundation the rest + stands on. +- **Next** — Ship pillar 2 (docs-vs-reality drift) on top of the existing + capture + diff + Action, since accuracy (3) is what makes its signal + trustworthy. +- **Later** — Broaden pillar 4 (export/integrations into the docs & SDK + ecosystem) and extend framework/language coverage where it serves the wedge. + +## How we measure the direction is working + +- Regulated / on-prem teams adopting *because* traffic never leaves — a segment + the SaaS camp cannot serve. +- Drift detection surfacing real doc-vs-reality divergences that spec-to-spec + tools structurally cannot see. +- A published, defensible cross-provider accuracy number. +- EasyDocs specs feeding downstream tools (Scalar/ReadMe/Speakeasy/Fern) as the + common upstream. diff --git a/apps/dashboard/src/app/api/drift/route.ts b/apps/dashboard/src/app/api/drift/route.ts new file mode 100644 index 0000000..cf8584f --- /dev/null +++ b/apps/dashboard/src/app/api/drift/route.ts @@ -0,0 +1,7 @@ +import { NextRequest, NextResponse } from 'next/server' +import { fetchDrift } from '@/lib/db' + +export async function GET(req: NextRequest) { + const project = req.nextUrl.searchParams.get('project') ?? undefined + return NextResponse.json(await fetchDrift(project)) +} diff --git a/apps/dashboard/src/components/Dashboard.tsx b/apps/dashboard/src/components/Dashboard.tsx index 61c7376..54f39d2 100644 --- a/apps/dashboard/src/components/Dashboard.tsx +++ b/apps/dashboard/src/components/Dashboard.tsx @@ -1,10 +1,14 @@ 'use client' -import { useState, useMemo } from 'react' +import { useState, useMemo, useEffect } from 'react' import { useRouter } from 'next/navigation' import { MethodBadge } from './MethodBadge' import { EndpointDetail } from './EndpointDetail' +import { DriftSummary } from './DriftSummary' +import { RedactionAudit, type AuditItem } from './RedactionAudit' +import { collectSensitiveFields } from '@easydocs/core/privacy/audit' import type { Endpoint, Project } from '@easydocs/core/schema' +import type { DriftData } from '@/lib/db' interface Props { endpoints: Endpoint[] @@ -16,6 +20,28 @@ export function Dashboard({ endpoints, projects, currentProject }: Props) { const router = useRouter() const [selected, setSelected] = useState(endpoints[0] ?? null) const [search, setSearch] = useState('') + const [drift, setDrift] = useState(null) + const [panel, setPanel] = useState<'drift' | 'audit' | null>(null) + + const audit = useMemo(() => { + const items: AuditItem[] = [] + for (const e of endpoints) { + const spec = e.isManuallyEdited && e.manualSpec ? e.manualSpec : e.spec + const fields = spec ? collectSensitiveFields(spec) : [] + if (fields.length > 0) items.push({ endpoint: e, fields }) + } + return { items, total: items.reduce((s, x) => s + x.fields.length, 0) } + }, [endpoints]) + + useEffect(() => { + const q = currentProject ? `?project=${currentProject}` : '' + let active = true + fetch(`/api/drift${q}`) + .then((r) => r.json()) + .then((d: DriftData) => { if (active) setDrift(d) }) + .catch(() => { /* drift is best-effort; ignore fetch errors */ }) + return () => { active = false } + }, [currentProject]) const filtered = useMemo( () => @@ -54,6 +80,42 @@ export function Dashboard({ endpoints, projects, currentProject }: Props) { {endpoints.length} endpoints + {drift?.configured && ( + + )} + + {audit.total > 0 && ( + + )} + {projects.length > 1 && (