From 2cd9b48ba9b9deda95d9791abf389dd1014d880e Mon Sep 17 00:00:00 2001 From: clippy Date: Thu, 30 Jul 2026 15:06:47 -0700 Subject: [PATCH 1/7] feat(embed): add the browser client (client + kit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the two remaining @monad-inc/embed entrypoints, typed to the /embed contract (connect is already on main): - src/client — the 9-route browser client (createEmbedClient) host front-ends call: catalog, connectors, session, pipeline build/status/state/remove. - src/kit — the backend-only Monad client (session, catalog, pipeline, teams) hosts/routers use server-side. Wires the ./client and ./kit subpath exports and the tsup/tsconfig build, and finalizes the shared eslint config in one place (ignore .venv, grant node globals to .mjs/.cjs scripts, allow _-prefixed unused args) so downstream router/conformance layers don't re-touch it. 19 unit tests, green against mocks. Layer 2 (PRO-401) of the staged rollout; stacked on the contract (PRO-400). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_011ASKA2VrufrNLGGrcFjY7n --- eslint.config.js | 26 ++- packages/embed/package.json | 20 ++ packages/embed/src/client/index.ts | 344 +++++++++++++++++++++++++++++ packages/embed/src/kit/catalog.ts | 55 +++++ packages/embed/src/kit/client.ts | 50 +++++ packages/embed/src/kit/index.ts | 167 ++++++++++++++ packages/embed/src/kit/pipeline.ts | 245 ++++++++++++++++++++ packages/embed/src/kit/session.ts | 52 +++++ packages/embed/src/kit/teams.ts | 23 ++ packages/embed/test/client.test.ts | 258 ++++++++++++++++++++++ packages/embed/test/kit.test.ts | 140 ++++++++++++ packages/embed/tsconfig.json | 7 +- packages/embed/tsup.config.ts | 6 +- 13 files changed, 1386 insertions(+), 7 deletions(-) create mode 100644 packages/embed/src/client/index.ts create mode 100644 packages/embed/src/kit/catalog.ts create mode 100644 packages/embed/src/kit/client.ts create mode 100644 packages/embed/src/kit/index.ts create mode 100644 packages/embed/src/kit/pipeline.ts create mode 100644 packages/embed/src/kit/session.ts create mode 100644 packages/embed/src/kit/teams.ts create mode 100644 packages/embed/test/client.test.ts create mode 100644 packages/embed/test/kit.test.ts diff --git a/eslint.config.js b/eslint.config.js index 7afc4df..e271cbb 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -4,9 +4,26 @@ import tseslint from 'typescript-eslint'; // Flat config for a plain-TypeScript monorepo. Deliberately NOT the Svelte // config the ui repo uses — nothing here is Svelte. export default tseslint.config( - { ignores: ['**/dist/**', '**/node_modules/**', '**/*.config.*', '**/.changeset/**'] }, + { + ignores: [ + '**/dist/**', + '**/node_modules/**', + '**/*.config.*', + '**/.changeset/**', + // Python virtualenvs (conformance harness, python router) ship vendored + // JS assets we don't lint. + '**/.venv/**' + ] + }, js.configs.recommended, ...tseslint.configs.recommended, + { + // Plain Node scripts (e.g. conformance boot scripts) — grant Node globals. + files: ['**/*.mjs', '**/*.cjs'], + languageOptions: { + globals: { process: 'readonly', console: 'readonly', Buffer: 'readonly' } + } + }, { files: ['**/*.ts'], languageOptions: { @@ -18,7 +35,12 @@ export default tseslint.config( rules: { // The SDK deliberately treats Monad API responses as untyped JSON // (see MonadRequest in lifecycle.ts). Surface but don't block on it. - '@typescript-eslint/no-explicit-any': 'warn' + '@typescript-eslint/no-explicit-any': 'warn', + // Allow intentionally-unused `_`-prefixed args (e.g. typed mock params). + '@typescript-eslint/no-unused-vars': [ + 'error', + { argsIgnorePattern: '^_', varsIgnorePattern: '^_' } + ] } } ); diff --git a/packages/embed/package.json b/packages/embed/package.json index c29b448..68b2eab 100644 --- a/packages/embed/package.json +++ b/packages/embed/package.json @@ -31,6 +31,26 @@ "default": "./dist/connect/index.cjs" } }, + "./kit": { + "import": { + "types": "./dist/kit/index.d.ts", + "default": "./dist/kit/index.js" + }, + "require": { + "types": "./dist/kit/index.d.cts", + "default": "./dist/kit/index.cjs" + } + }, + "./client": { + "import": { + "types": "./dist/client/index.d.ts", + "default": "./dist/client/index.js" + }, + "require": { + "types": "./dist/client/index.d.cts", + "default": "./dist/client/index.cjs" + } + }, "./package.json": "./package.json" }, "sideEffects": false, diff --git a/packages/embed/src/client/index.ts b/packages/embed/src/client/index.ts new file mode 100644 index 0000000..8125553 --- /dev/null +++ b/packages/embed/src/client/index.ts @@ -0,0 +1,344 @@ +/** + * @monad-inc/embed/client — the browser-side client for the `/embed` routes. + * + * This is the frontend counterpart to the backend routers (`@monad-inc/embed-server` + * and the Go/Python ports): both speak the same `/embed` contract. Once you've + * mounted a router on your backend, this client calls those routes from the + * browser so you don't hand-roll `fetch('/embed/…')` calls or the + * mint-session-then-mount-the-form dance. + * + * import { createEmbedClient } from '@monad-inc/embed/client'; + * + * const monad = createEmbedClient(); // defaults to your '/embed' base path + * const catalog = await monad.listCatalog('input'); + * + * // one call: mint a session, fetch the frame origin, mount the form + * card.onclick = () => + * monad.openConnectorForm({ + * container: '#connector-modal', + * kind: 'input', + * typeId: type.typeId, + * onSave: ({ id, name }) => monad.buildIngress(id, name ?? type.name), + * }); + * + * The low-level iframe primitive (`createConnectorFrame`) lives in + * `@monad-inc/embed/connect`; this client wraps it. It never sees your API key — + * that stays behind your `/embed` routes. + */ +import { + createConnectorFrame, + type Appearance, + type ComponentKind, + type ConnectorFrame +} from '../connect'; + +export type { Appearance, ComponentKind, ConnectorFrame } from '../connect'; + +/* ===== contract response shapes (camelCase, mirroring the /embed spec) ===== */ + +export interface EmbedConfigResponse { + frameOrigin: string; + apiBase: string; +} + +export interface EmbedSession { + sessionToken: string; + organizationId: string; + expiresAt: string; +} + +export interface CatalogType { + typeId: string; + name: string; +} + +export interface ConfiguredConnector { + id: string; + typeId: string; + name: string; +} + +export interface BuiltPipeline { + pipelineId: string; + outputId: string; + status: string; + active: boolean; +} + +export interface PipelineStatus { + hasPipeline: boolean; + enabled: boolean; + pipelineId?: string; + inputId?: string; + outputId?: string; +} + +/** Thrown on a non-2xx from an `/embed` route, carrying the contract's error model. */ +export class EmbedRequestError extends Error { + constructor( + public readonly status: number, + public readonly code: string, + message: string + ) { + super(message); + this.name = 'EmbedRequestError'; + } +} + +export interface EmbedClientOptions { + /** Where your backend mounted the routes. Defaults to `/embed`. */ + basePath?: string; + /** Passed to every `fetch`. Defaults to `'same-origin'` (sends your cookies). */ + credentials?: RequestCredentials; + /** Extra headers on every request — e.g. a CSRF token. */ + headers?: Record; +} + +export interface OpenConnectorFormOptions { + /** DOM node (or selector) the iframe is appended to. */ + container: HTMLElement | string; + /** `'input'` (a source) or `'output'` (a destination). */ + kind: ComponentKind; + /** Connector type slug, e.g. `'aws-cloudtrail'`. */ + typeId: string; + /** + * Name for the created connector. Omit and the form appends a unique suffix + * to the type name, so two connectors of the same type don't collide. + */ + name?: string; + /** Title shown in the iframe header. */ + displayName?: string; + /** Description for the created connector (create only). */ + description?: string; + /** Let the end user edit the name inside the iframe (default off). */ + isNameEditable?: boolean; + /** Let the end user edit the description inside the iframe (default off). */ + isDescriptionEditable?: boolean; + /** Pass to edit an existing connector; omit to create a new one. */ + existingId?: string; + /** Exposes Monad's internal synthetic-data toggle. Keep off in production. */ + synthetic?: boolean; + /** Theming tokens applied inside the iframe. */ + appearance?: Appearance; + /** Full-CSS escape hatch (all-or-nothing). */ + stylesheets?: string[]; + /** + * Destroy the iframe automatically when the user saves or cancels. Default + * `true` — your `onSave`/`onCancel` run after teardown. Set `false` to manage + * the returned handle's lifecycle yourself. + */ + autoDestroy?: boolean; + /** + * After a successful save, build the pipeline too — the direction follows + * `kind` (input → ingress, output → egress). Collapses "open the form" and + * "turn the connector into a running pipeline" into one declaration. Default + * `false`. Handle the result with `onConnected` / `onBuildError`. + * + * Important: the form can save while the *build* fails, leaving a connector + * with no pipeline behind it. That failure goes to `onBuildError` (with a + * ready-made `retry`), never swallowed — so you can offer a "finish setup". + */ + autoBuild?: boolean; + /** Fired with the running pipeline when `autoBuild` succeeds. */ + onConnected?: (pipeline: BuiltPipeline) => void; + /** + * Fired when `autoBuild`'s build fails. `retry` re-attempts the same build + * (same connector) and again resolves to `onConnected` / `onBuildError`. + */ + onBuildError?: (error: EmbedRequestError, retry: () => Promise) => void; + /** Fired when the user saves successfully. */ + onSave?: (connector: { id: string; name?: string }) => void; + /** Fired when the user cancels. */ + onCancel?: () => void; + /** Fired on a save/test/load failure inside the iframe. */ + onError?: (message: string) => void; +} + +/** A browser client bound to your `/embed` routes. */ +export interface EmbedClient { + /** `GET /embed/config` — the iframe origin + API base (cached after first call). */ + getConfig(): Promise; + /** `POST /embed/session` — a fresh short-lived, team-scoped token. */ + mintSession(): Promise; + /** `GET /embed/catalog` — the connector types you offer for a kind. */ + listCatalog(kind: ComponentKind): Promise; + /** `GET /embed/connectors` — the connectors this tenant has configured. */ + listConnectors(kind: ComponentKind): Promise; + /** + * Mint a session, resolve the frame origin, and mount the connector form — + * the whole "open the form" flow in one call. Returns the frame handle. + */ + openConnectorForm(opts: OpenConnectorFormOptions): Promise; + /** + * Build the pipeline for a configured connector — direction follows `kind` + * (input → ingress, output → egress), so you don't pick a route. Sugar over + * `buildIngress` / `buildEgress`. + */ + buildPipeline(opts: { + connectorId: string; + kind: ComponentKind; + name: string; + }): Promise; + /** `POST /embed/pipelines/ingress` — wire a configured input into a running pipeline. */ + buildIngress(inputId: string, name: string): Promise; + /** `POST /embed/pipelines/egress` — wire your source into the configured output. */ + buildEgress(outputId: string, name: string): Promise; + /** `GET /embed/pipelines` — resolve a connector's pipeline + enabled state. */ + pipelineStatus(connectorId: string, kind: ComponentKind): Promise; + /** `POST /embed/pipelines/state` — enable or disable without deleting config. */ + setPipelineState(pipelineId: string, enabled: boolean): Promise; + /** `POST /embed/pipelines/remove` — tear down the integration behind a connector. */ + removeIntegration(connectorId: string, kind: ComponentKind): Promise; + /** The vendor logo URL for a connector type, from Monad's icon endpoint. */ + iconUrl(typeId: string): Promise; +} + +const enc = encodeURIComponent; + +/** Create a client for the `/embed` routes your backend mounted. */ +export function createEmbedClient(options: EmbedClientOptions = {}): EmbedClient { + const basePath = (options.basePath ?? '/embed').replace(/\/$/, ''); + const credentials = options.credentials ?? 'same-origin'; + const baseHeaders = options.headers ?? {}; + let configPromise: Promise | undefined; + + async function req(method: string, path: string, body?: unknown): Promise { + const headers: Record = { ...baseHeaders }; + const init: RequestInit = { method, credentials, headers }; + if (body !== undefined) { + headers['Content-Type'] = 'application/json'; + init.body = JSON.stringify(body); + } + const r = await fetch(`${basePath}${path}`, init); + if (!r.ok) { + let payload: { code?: string; message?: string } | undefined; + try { + payload = await r.json(); + } catch { + // non-JSON error body — fall back to a generic message + } + throw new EmbedRequestError( + r.status, + payload?.code ?? 'request_failed', + payload?.message ?? `${method} ${path} failed with ${r.status}` + ); + } + if (r.status === 204) return undefined as T; + const text = await r.text(); + if (!text) return undefined as T; + try { + return JSON.parse(text) as T; + } catch { + // A 2xx with a non-JSON body — typically an auth redirect / login HTML + // page served because the host session expired. Surface it as a + // branded EmbedRequestError instead of a raw SyntaxError. + throw new EmbedRequestError( + r.status, + 'invalid_response', + `${method} ${path} returned a non-JSON response (status ${r.status}); the host session may have expired.` + ); + } + } + + const client: EmbedClient = { + getConfig() { + return (configPromise ??= req('GET', '/config')); + }, + mintSession() { + return req('POST', '/session'); + }, + listCatalog(kind) { + return req('GET', `/catalog?kind=${enc(kind)}`); + }, + listConnectors(kind) { + return req('GET', `/connectors?kind=${enc(kind)}`); + }, + buildPipeline({ connectorId, kind, name }) { + return kind === 'input' + ? client.buildIngress(connectorId, name) + : client.buildEgress(connectorId, name); + }, + buildIngress(inputId, name) { + return req('POST', '/pipelines/ingress', { inputId, name }); + }, + buildEgress(outputId, name) { + return req('POST', '/pipelines/egress', { outputId, name }); + }, + pipelineStatus(connectorId, kind) { + return req( + 'GET', + `/pipelines?connectorId=${enc(connectorId)}&kind=${enc(kind)}` + ); + }, + setPipelineState(pipelineId, enabled) { + return req('POST', '/pipelines/state', { pipelineId, enabled }); + }, + removeIntegration(connectorId, kind) { + return req('POST', '/pipelines/remove', { connectorId, kind }); + }, + async iconUrl(typeId) { + const { frameOrigin } = await client.getConfig(); + return `${new URL(frameOrigin).origin}/external/icons/raw/${enc(typeId)}.svg`; + }, + async openConnectorForm(opts) { + const [config, session] = await Promise.all([client.getConfig(), client.mintSession()]); + const autoDestroy = opts.autoDestroy ?? true; + const frame: ConnectorFrame = createConnectorFrame({ + container: opts.container, + frameOrigin: config.frameOrigin, + apiBase: config.apiBase, + sessionToken: session.sessionToken, + organizationId: session.organizationId, + kind: opts.kind, + typeId: opts.typeId, + name: opts.name, + displayName: opts.displayName, + description: opts.description, + isNameEditable: opts.isNameEditable, + isDescriptionEditable: opts.isDescriptionEditable, + existingId: opts.existingId, + synthetic: opts.synthetic, + appearance: opts.appearance, + stylesheets: opts.stylesheets, + onSave: (connector) => { + if (autoDestroy) frame.destroy(); + opts.onSave?.(connector); + if (opts.autoBuild) { + const name = connector.name ?? opts.name ?? opts.typeId; + const runBuild = async (): Promise => { + try { + const pipeline = await client.buildPipeline({ + connectorId: connector.id, + kind: opts.kind, + name + }); + opts.onConnected?.(pipeline); + } catch (e) { + const err = + e instanceof EmbedRequestError + ? e + : new EmbedRequestError( + 0, + 'build_failed', + e instanceof Error ? e.message : 'Pipeline build failed.' + ); + // Never swallow a build failure — the connector exists with no + // pipeline. Route it to onBuildError (with retry), else onError. + if (opts.onBuildError) opts.onBuildError(err, runBuild); + else opts.onError?.(err.message); + } + }; + void runBuild(); + } + }, + onCancel: () => { + if (autoDestroy) frame.destroy(); + opts.onCancel?.(); + }, + onError: opts.onError + }); + return frame; + } + }; + return client; +} diff --git a/packages/embed/src/kit/catalog.ts b/packages/embed/src/kit/catalog.ts new file mode 100644 index 0000000..f8f09fc --- /dev/null +++ b/packages/embed/src/kit/catalog.ts @@ -0,0 +1,55 @@ +/** + * The catalog: what connector types you can offer, and what a tenant has + * already configured. Both are plain Monad API reads behind your backend. + */ +import type { ComponentKind } from '../connect'; +import type { MonadConfig } from './client'; +import { monadRequest } from './client'; + +/** A connector type available to embed, e.g. `{ type_id: 'aws-cloudtrail', name: 'AWS CloudTrail' }`. */ +export interface CatalogType { + type_id: string; + name: string; +} + +/** A connector a tenant has already configured. */ +export interface ConfiguredConnector { + id: string; + /** Connector type slug. Normalized from the API's `type` field (see listConnectors). */ + type_id: string; + name: string; +} + +/** + * The connector types available for a kind ('input' | 'output'). Pass `allow` + * to expose only the type_ids your product supports — the host-curated + * allow-list. A real product rarely surfaces all 300+ connectors, and that + * decision is yours to make server-side, not the end user's. + */ +export async function listCatalog( + cfg: MonadConfig, + kind: ComponentKind, + allow?: string[] +): Promise { + const types = (await monadRequest(cfg)(`/v1/${kind}s`)) as CatalogType[]; + if (!allow || allow.length === 0) return types; + const allowed = new Set(allow); + return types.filter((t) => allowed.has(t.type_id)); +} + +/** The connectors a tenant has already configured, so you can show + manage them. */ +export async function listConnectors( + cfg: MonadConfig, + org: string, + kind: ComponentKind +): Promise { + // Configured rows come back wrapped as { inputs: [...] } / { outputs: [...] }, + // and carry the type slug as `type` — NOT `type_id`, which the catalog uses. + // Normalize to `type_id` so the rest of your app has one consistent field. + const page = (await monadRequest(cfg)(`/v1/${org}/${kind}s?limit=1000&offset=0`)) as Record< + string, + { id: string; type: string; name: string }[] + >; + const rows = page[`${kind}s`] ?? []; + return rows.map((r) => ({ id: r.id, type_id: r.type, name: r.name })); +} diff --git a/packages/embed/src/kit/client.ts b/packages/embed/src/kit/client.ts new file mode 100644 index 0000000..d5ba8a3 --- /dev/null +++ b/packages/embed/src/kit/client.ts @@ -0,0 +1,50 @@ +/** + * @monad-inc/embed/kit — the backend glue every embed integration needs + * beyond the iframe + lifecycle helpers in `@monad-inc/embed/connect`. + * + * The connect module gives you two things: `createConnectorFrame()` (mount + * the secure form) and a set of loose lifecycle helpers that each re-take a + * `{ request, org, … }` bag. The kit closes the remaining gap: it binds your + * API key + base ONCE into a small client, then exposes plain methods — + * `mintSession`, `listCatalog`, `connectSource`, `connectDestination`, + * status/enable/remove — so your route handlers read like the operation they + * perform, not like plumbing. + * + * Everything here is backend-only: it carries your long-lived API key, so it + * must never run in the browser. The one browser-side call (mounting the + * iframe) still lives in `@monad-inc/embed/connect`. + */ +import type { MonadRequest } from '../connect'; + +/** Everything the backend half needs to talk to Monad. */ +export interface MonadConfig { + /** Long-lived Monad API key (JWT). Server-side only — never ship to the browser. */ + apiKey: string; + /** Monad API base. Defaults to "https://app.monad.com/api" (production); set only for non-prod. */ + apiBase?: string; +} + +/** + * Build the authenticated request thunk the connect lifecycle helpers expect. + * Every call carries your API key, so this only ever runs on your backend. + * Exposed so you can drop down to a raw Monad call the kit doesn't wrap yet: + * `monad.request('/v2/{org}/…', { method: 'POST', body })`. + */ +export function monadRequest(cfg: MonadConfig): MonadRequest { + const apiBase = cfg.apiBase ?? 'https://app.monad.com/api'; + return async (path, init) => { + const r = await fetch(`${apiBase}${path}`, { + ...init, + headers: { + 'Content-Type': 'application/json', + Authorization: `ApiKey ${cfg.apiKey}`, + ...(init?.headers ?? {}) + } + }); + if (!r.ok) { + throw new Error(`${r.status} ${path}: ${(await r.text()).slice(0, 300)}`); + } + const text = await r.text(); + return text ? JSON.parse(text) : undefined; + }; +} diff --git a/packages/embed/src/kit/index.ts b/packages/embed/src/kit/index.ts new file mode 100644 index 0000000..df8dfcb --- /dev/null +++ b/packages/embed/src/kit/index.ts @@ -0,0 +1,167 @@ +/** + * @monad-inc/embed/kit — the backend companion to `@monad-inc/embed/connect`. + * + * `connect` mounts the secure iframe and exposes low-level lifecycle helpers. + * `kit` is the ergonomic layer on top: bind your API key + base ONCE with + * `createEmbedClient`, then call plain methods. Scope to a tenant with + * `.team(org)` and the org drops out of every subsequent call. + * + * import { createEmbedClient } from '@monad-inc/embed/kit'; + * + * const monad = createEmbedClient({ apiKey: process.env.MONAD_API_KEY!, apiBase }); + * const team = monad.team(tenant.orgId); + * + * // backend route: hand a session to the browser + * const session = await team.mintSession(); // → { sessionToken, organizationId, expiresAt } + * + * // backend route: the iframe returned an input id — make data flow + * const built = await team.connectSource({ inputId, name: 'AWS CloudTrail' }); + * + * Every function is also exported standalone (taking an explicit `cfg` + + * `org`) if you'd rather not hold a client. The types are exported too, so you + * import them instead of copying a mini-library out of a demo. + */ +import type { MonadRequest, BuiltPipeline, CleanupPolicy, ComponentKind } from '../connect'; +import type { MonadConfig } from './client'; +import { monadRequest } from './client'; +import { mintSession, type EmbedSession } from './session'; +import { listCatalog, listConnectors, type CatalogType, type ConfiguredConnector } from './catalog'; +import { + wirePipeline, + connectSource, + connectDestination, + pipelineStatus, + findPipelineByOutput, + setEnabled, + remove, + removeIngress, + removeEgress, + type ResolvedEgress +} from './pipeline'; +import { provisionTeam } from './teams'; + +/* ===== re-exports: config-free functions + all types ===== */ + +export type { MonadConfig } from './client'; +export { monadRequest } from './client'; +export type { EmbedSession } from './session'; +export { mintSession } from './session'; +export type { CatalogType, ConfiguredConnector } from './catalog'; +export { listCatalog, listConnectors } from './catalog'; +export type { ResolvedEgress } from './pipeline'; +export { + wirePipeline, + connectSource, + connectDestination, + pipelineStatus, + findPipelineByOutput, + setEnabled, + remove, + removeIngress, + removeEgress +} from './pipeline'; +export { provisionTeam } from './teams'; + +// Convenience re-exports so a kit user gets the pipeline result type + cleanup +// presets without also reaching into '@monad-inc/embed/connect'. +export { + CLEANUP_FULL, + CLEANUP_KEEP_OUTPUT, + CLEANUP_PIPELINE_ONLY, + type BuiltPipeline, + type CleanupPolicy, + type ComponentKind, + type MonadRequest +} from '../connect'; + +/* ===== org-scoped client (the common case) ===== */ + +type PipelineStatus = Awaited>; + +/** + * A client bound to one tenant's team (org). Every method that needed an `org` + * has it baked in — the shape a backend request handler actually wants, since + * a request is always in one tenant's context. + */ +export interface TeamClient { + /** The org this handle is scoped to. */ + readonly org: string; + mintSession(ttlSeconds?: number): Promise; + listConnectors(kind: ComponentKind): Promise; + wirePipeline(opts: { inputId: string; outputId: string; name: string }): Promise; + /** Ingress: user configured an input → wire to your store, or dev/null if omitted. */ + connectSource(opts: { + inputId: string; + name: string; + toOutputId?: string; + }): Promise; + /** Egress: user configured an output → wire your pre-provisioned source to it. */ + connectDestination(opts: { + outputId: string; + name: string; + fromInputId: string; + }): Promise; + /** Resolve a pipeline + enabled state from an input id. */ + pipelineStatus(inputId: string): Promise; + /** Resolve an egress pipeline from an output id. */ + findPipelineByOutput(outputId: string): Promise; + setEnabled(opts: { pipelineId: string; enabled: boolean }): Promise; + remove(opts: { + pipelineId?: string; + inputId?: string; + outputId?: string; + cleanup?: CleanupPolicy; + }): Promise; + removeIngress(opts: { + pipelineId?: string; + inputId?: string; + outputId?: string; + keepStore: boolean; + }): Promise; + removeEgress(opts: { pipelineId?: string; outputId?: string }): Promise; +} + +/** A client bound to your Monad config. `.team(org)` scopes it to a tenant. */ +export interface EmbedClient { + readonly config: MonadConfig; + /** The raw authenticated request thunk, for calls the kit doesn't wrap yet. */ + readonly request: MonadRequest; + /** Step 0: create a new tenant's team under your org. */ + provisionTeam(opts: { parentOrgId: string; name: string; friendlyName: string }): Promise<{ + id: string; + }>; + /** The connector catalog for a kind, optionally filtered to your allow-list. */ + listCatalog(kind: ComponentKind, allow?: string[]): Promise; + /** Mint a session scoped to `org`. */ + mintSession(org: string, ttlSeconds?: number): Promise; + /** Bind every org-scoped method to one tenant. */ + team(org: string): TeamClient; +} + +/** + * Create a client bound to your Monad config. Hold one per process (your API + * key doesn't change) and call `.team(org)` per request. + */ +export function createEmbedClient(config: MonadConfig): EmbedClient { + return { + config, + request: monadRequest(config), + provisionTeam: (opts) => provisionTeam(config, opts), + listCatalog: (kind, allow) => listCatalog(config, kind, allow), + mintSession: (org, ttlSeconds) => mintSession(config, org, ttlSeconds), + team: (org) => ({ + org, + mintSession: (ttlSeconds) => mintSession(config, org, ttlSeconds), + listConnectors: (kind) => listConnectors(config, org, kind), + wirePipeline: (opts) => wirePipeline(config, { org, ...opts }), + connectSource: (opts) => connectSource(config, { org, ...opts }), + connectDestination: (opts) => connectDestination(config, { org, ...opts }), + pipelineStatus: (inputId) => pipelineStatus(config, { org, inputId }), + findPipelineByOutput: (outputId) => findPipelineByOutput(config, { org, outputId }), + setEnabled: (opts) => setEnabled(config, { org, ...opts }), + remove: (opts) => remove(config, { org, ...opts }), + removeIngress: (opts) => removeIngress(config, { org, ...opts }), + removeEgress: (opts) => removeEgress(config, { org, ...opts }) + }) + }; +} diff --git a/packages/embed/src/kit/pipeline.ts b/packages/embed/src/kit/pipeline.ts new file mode 100644 index 0000000..0c821cb --- /dev/null +++ b/packages/embed/src/kit/pipeline.ts @@ -0,0 +1,245 @@ +/** + * Pipeline bootstrap + management — what happens after a user configures a + * connector in the iframe, and how you manage the integration afterward. + * + * The iframe hands your backend only an id: an INPUT id when the user + * configured a source, an OUTPUT id when they configured a destination. A bare + * connector moves no data — it needs a pipeline. These helpers stand that + * pipeline up and manage it, so a customer never has to hand-assemble a + * nodes/edges payload. + * + * Two directions, one primitive: + * • Ingress — the user configured an INPUT; wire it → your store (or a + * throwaway dev/null sink). → connectSource() + * • Egress — the user configured an OUTPUT; wire your pre-provisioned + * source → it. → connectDestination() + * Both are `wirePipeline` with the pre-provisioned side flipped. + */ +import { + buildDevNullPipeline, + findIntegrationPipeline, + setIntegrationEnabled, + deleteIntegration, + CLEANUP_FULL, + CLEANUP_KEEP_OUTPUT, + type BuiltPipeline, + type CleanupPolicy +} from '../connect'; +import type { MonadConfig } from './client'; +import { monadRequest } from './client'; + +/** How long `wirePipeline` waits for a new pipeline to report Running. */ +const POLL_ATTEMPTS = 15; +const POLL_INTERVAL_MS = 2000; + +/** + * Wire one input → one output and poll until it reports Running. The shared + * primitive behind both directions. The connect module's `buildDevNullPipeline` + * is the special case that also creates a throwaway sink; this is the general + * "wire two existing connectors" case the package otherwise lacks. + */ +export async function wirePipeline( + cfg: MonadConfig, + opts: { org: string; inputId: string; outputId: string; name: string } +): Promise { + const { org, inputId, outputId, name } = opts; + const req = monadRequest(cfg); + const pipeline = (await req(`/v2/${org}/pipelines/`, { + method: 'POST', + body: JSON.stringify({ + name, + description: 'Created when the connector was configured via embed', + enabled: true, + nodes: [ + { slug: 'in', component_id: inputId, component_type: 'input', enabled: true }, + { slug: 'out', component_id: outputId, component_type: 'output', enabled: true } + ], + edges: [ + { + from_node_instance_id: 'in', + to_node_instance_id: 'out', + description: 'all records', + conditions: { operator: 'always' } + } + ] + }) + })) as { id: string }; + + let status = 'Pending'; + for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { + const s = (await req(`/v2/${org}/pipelines/${pipeline.id}/status`)) as { status?: string }; + status = s?.status ?? 'Unknown'; + if (status === 'Running' || status === 'Erroring') break; + await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); + } + return { pipelineId: pipeline.id, outputId, status, active: status === 'Running' }; +} + +/** + * Ingress. After a user configures an INPUT, stand up a pipeline so data flows. + * Pass `toOutputId` (a destination you pre-provisioned, one per tenant) to send + * to your real store; omit it and a throwaway dev/null sink is created — the + * zero-setup default that lets you demo end-to-end before you have a store. + */ +export function connectSource( + cfg: MonadConfig, + opts: { org: string; inputId: string; name: string; toOutputId?: string } +): Promise { + const { org, inputId, name, toOutputId } = opts; + if (toOutputId) return wirePipeline(cfg, { org, inputId, outputId: toOutputId, name }); + return buildDevNullPipeline({ request: monadRequest(cfg), org, inputId, inputName: name }); +} + +/** + * Egress — the mirror of connectSource. After a user configures an OUTPUT, wire + * your PRE-PROVISIONED source (your product, one per tenant team) → it. One + * source fans out to many destinations. `fromInputId` is required: unlike + * ingress there's no throwaway default, because egress by definition needs a + * real source to read from. + */ +export function connectDestination( + cfg: MonadConfig, + opts: { org: string; outputId: string; name: string; fromInputId: string } +): Promise { + const { org, outputId, name, fromInputId } = opts; + return wirePipeline(cfg, { org, inputId: fromInputId, outputId, name }); +} + +/** Resolve an integration's pipeline + enabled state, knowing only the input id. */ +export async function pipelineStatus( + cfg: MonadConfig, + opts: { org: string; inputId: string } +): Promise<{ hasPipeline: boolean; enabled: boolean; pipelineId?: string; outputId?: string }> { + const resolved = await findIntegrationPipeline({ + request: monadRequest(cfg), + org: opts.org, + inputId: opts.inputId + }); + return { + hasPipeline: resolved !== null, + enabled: resolved?.enabled ?? false, + pipelineId: resolved?.pipelineId, + outputId: resolved?.outputId + }; +} + +interface PipelineNode { + component_type: string; + component_id: string; +} +interface PipelineSummary { + id: string; + enabled?: boolean; +} + +/** Where an egress destination's pipeline lives and its current state. */ +export interface ResolvedEgress { + pipelineId: string; + enabled: boolean; + outputId: string; + inputId?: string; +} + +/** + * Resolve the pipeline feeding a given OUTPUT — the egress counterpart to the + * connect module's `findIntegrationPipeline` (which keys on input id). Walks + * the org's pipelines and matches the output node, so you can manage a + * destination knowing only its output id. Returns null when nothing feeds it. + */ +export async function findPipelineByOutput( + cfg: MonadConfig, + opts: { org: string; outputId: string } +): Promise { + const { org, outputId } = opts; + const req = monadRequest(cfg); + const listed = await req(`/v2/${org}/pipelines/`); + const pipelines: PipelineSummary[] = Array.isArray(listed) + ? listed + : (listed?.pipelines ?? listed?.data ?? []); + for (const summary of pipelines) { + if (!summary?.id) continue; + let detail; + try { + detail = await req(`/v2/${org}/pipelines/${summary.id}`); + } catch { + continue; // skip a pipeline we can't read rather than failing the whole scan + } + const p = detail?.config ?? detail?.pipeline ?? detail ?? {}; + const nodes: PipelineNode[] = p.nodes ?? []; + const out = nodes.find((n) => n.component_type === 'output' && n.component_id === outputId); + if (!out) continue; + const input = nodes.find((n) => n.component_type === 'input'); + return { + pipelineId: summary.id, + enabled: Boolean(p.enabled ?? summary.enabled), + outputId, + inputId: input?.component_id + }; + } + return null; +} + +/** Stop / resume data flow without deleting any config. */ +export function setEnabled( + cfg: MonadConfig, + opts: { org: string; pipelineId: string; enabled: boolean } +): Promise { + return setIntegrationEnabled({ request: monadRequest(cfg), ...opts }); +} + +/** + * Remove an integration. `cleanup` decides what's torn down vs kept — pass a + * connect-module preset (CLEANUP_FULL, CLEANUP_KEEP_OUTPUT, …) or your own + * `{ pipeline?, input?, output? }`. Only ids you supply are ever touched. + */ +export function remove( + cfg: MonadConfig, + opts: { + org: string; + pipelineId?: string; + inputId?: string; + outputId?: string; + cleanup?: CleanupPolicy; + } +): Promise { + const { org, pipelineId, inputId, outputId, cleanup = CLEANUP_FULL } = opts; + return deleteIntegration({ + request: monadRequest(cfg), + org, + pipelineId, + inputId, + outputId, + cleanup + }); +} + +/** + * Remove an INGRESS integration with the right cleanup baked in. If the output + * is your pre-provisioned shared store, keep it (other inputs still feed it); + * if it's the auto-created dev/null sink, tear it down too. + */ +export function removeIngress( + cfg: MonadConfig, + opts: { + org: string; + pipelineId?: string; + inputId?: string; + outputId?: string; + /** True when the pipeline's output is your shared store (keep it). */ + keepStore: boolean; + } +): Promise { + const { keepStore, ...ids } = opts; + return remove(cfg, { ...ids, cleanup: keepStore ? CLEANUP_KEEP_OUTPUT : CLEANUP_FULL }); +} + +/** + * Remove an EGRESS destination: delete its pipeline + the output, but keep the + * shared source — other destinations still feed from it. + */ +export function removeEgress( + cfg: MonadConfig, + opts: { org: string; pipelineId?: string; outputId?: string } +): Promise { + return remove(cfg, { ...opts, cleanup: { pipeline: true, input: false, output: true } }); +} diff --git a/packages/embed/src/kit/session.ts b/packages/embed/src/kit/session.ts new file mode 100644 index 0000000..d614e35 --- /dev/null +++ b/packages/embed/src/kit/session.ts @@ -0,0 +1,52 @@ +/** + * Session minting — trade your long-lived API key for a short-lived, + * team-scoped token. The token (never the key) is what reaches the browser, + * and from there the iframe over postMessage — so it stays out of history and + * the Referer header. + */ +import type { MonadConfig } from './client'; +import { monadRequest } from './client'; + +/** + * A minted embed session, shaped to drop straight into `createConnectorFrame` + * / `mountConnector` with no field remapping. + * + * Monad's `POST /v3/sessions` returns snake_case (`session_token`, + * `expires_at`), but the iframe SDK expects `sessionToken` + + * `organizationId`. Every host was remapping this by hand at the mount site + * (see EMBED-GAPS #4). The kit does it once, here. + */ +export interface EmbedSession { + /** Pass straight to `createConnectorFrame({ sessionToken })`. */ + sessionToken: string; + /** The org the token is scoped to — pass to `createConnectorFrame({ organizationId })`. */ + organizationId: string; + /** ISO-8601 expiry, so you can decide whether to re-mint before rendering. */ + expiresAt: string; +} + +interface RawSession { + session_token: string; + expires_at: string; +} + +/** + * Mint an embed session scoped to one team (org). Call from your backend right + * before rendering the form, mapping your own tenant → its Monad team. Default + * TTL is 30 minutes. + */ +export async function mintSession( + cfg: MonadConfig, + organizationId: string, + ttlSeconds = 1800 +): Promise { + const raw = (await monadRequest(cfg)('/v3/sessions', { + method: 'POST', + body: JSON.stringify({ ttl_seconds: ttlSeconds, organization_id: organizationId }) + })) as RawSession; + return { + sessionToken: raw.session_token, + organizationId, + expiresAt: raw.expires_at + }; +} diff --git a/packages/embed/src/kit/teams.ts b/packages/embed/src/kit/teams.ts new file mode 100644 index 0000000..affd33f --- /dev/null +++ b/packages/embed/src/kit/teams.ts @@ -0,0 +1,23 @@ +/** + * Step 0 — per-customer teams. Each of your customers gets its own Monad team + * (a sub-organization) under your org: data isolation, per-tenant limits, and + * clean billing/audit boundaries. One API key, many teams — the multi-tenant + * foundation everything else is scoped to. + */ +import type { MonadConfig } from './client'; +import { monadRequest } from './client'; + +/** + * Create a team under your org and return its id. Store that id against your + * own tenant record, then scope all of that customer's connectors, pipelines, + * and sessions to it. + */ +export async function provisionTeam( + cfg: MonadConfig, + opts: { parentOrgId: string; name: string; friendlyName: string } +): Promise<{ id: string }> { + return (await monadRequest(cfg)(`/v3/${opts.parentOrgId}/organizations`, { + method: 'POST', + body: JSON.stringify({ name: opts.name, friendly_name: opts.friendlyName }) + })) as { id: string }; +} diff --git a/packages/embed/test/client.test.ts b/packages/embed/test/client.test.ts new file mode 100644 index 0000000..1ad1306 --- /dev/null +++ b/packages/embed/test/client.test.ts @@ -0,0 +1,258 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { ConnectorFrameOptions } from '../src/connect'; + +// Mock the low-level iframe primitive so we can assert how the client wires it, +// without needing a DOM. (The type-only import above is unaffected by the mock.) +const { createConnectorFrame, destroy } = vi.hoisted(() => { + const destroy = vi.fn(); + const createConnectorFrame = vi.fn((_opts: ConnectorFrameOptions) => ({ destroy })); + return { destroy, createConnectorFrame }; +}); +vi.mock('../src/connect', () => ({ createConnectorFrame })); + +import { createEmbedClient, EmbedRequestError } from '../src/client'; + +/** Route mocked fetch by "METHOD /path" → { status?, body }. Default 200. */ +function mockApi(routes: Record) { + const calls: { method: string; path: string; body?: unknown }[] = []; + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + const method = (init?.method ?? 'GET').toUpperCase(); + const body = init?.body ? JSON.parse(init.body as string) : undefined; + calls.push({ method, path: url, body }); + const route = routes[`${method} ${url}`]; + if (!route) throw new Error(`unmocked ${method} ${url}`); + const status = route.status ?? 200; + return { + ok: status >= 200 && status < 300, + status, + json: async () => route.body, + text: async () => (route.body === undefined ? '' : JSON.stringify(route.body)) + } as Response; + }); + vi.stubGlobal('fetch', fetchMock); + return calls; +} + +afterEach(() => { + vi.unstubAllGlobals(); + vi.clearAllMocks(); +}); + +describe('createEmbedClient', () => { + it('listCatalog calls the /embed route and returns the typed list', async () => { + mockApi({ + 'GET /embed/catalog?kind=input': { + body: [{ typeId: 'aws-cloudtrail', name: 'AWS' }] + } + }); + const rows = await createEmbedClient().listCatalog('input'); + expect(rows).toEqual([{ typeId: 'aws-cloudtrail', name: 'AWS' }]); + }); + + it('buildIngress POSTs { inputId, name }', async () => { + const calls = mockApi({ + 'POST /embed/pipelines/ingress': { + status: 201, + body: { pipelineId: 'p1', outputId: 'o1', status: 'Running', active: true } + } + }); + const built = await createEmbedClient().buildIngress('in_1', 'CloudTrail'); + expect(built.active).toBe(true); + expect(calls[0]?.body).toEqual({ inputId: 'in_1', name: 'CloudTrail' }); + }); + + it('buildPipeline routes by kind (input → ingress, output → egress)', async () => { + const pipe = { pipelineId: 'p', outputId: 'o', status: 'Running', active: true }; + const calls = mockApi({ + 'POST /embed/pipelines/ingress': { status: 201, body: pipe }, + 'POST /embed/pipelines/egress': { status: 201, body: pipe } + }); + const monad = createEmbedClient(); + await monad.buildPipeline({ connectorId: 'in_1', kind: 'input', name: 'CT' }); + await monad.buildPipeline({ connectorId: 'out_1', kind: 'output', name: 'Splunk' }); + expect(calls.map((c) => c.path)).toEqual([ + '/embed/pipelines/ingress', + '/embed/pipelines/egress' + ]); + expect(calls[0]?.body).toEqual({ inputId: 'in_1', name: 'CT' }); + expect(calls[1]?.body).toEqual({ outputId: 'out_1', name: 'Splunk' }); + }); + + it('honors a custom basePath', async () => { + const calls = mockApi({ 'POST /api/monad/session': { body: {} } }); + await createEmbedClient({ basePath: '/api/monad' }).mintSession(); + expect(calls[0]?.path).toBe('/api/monad/session'); + }); + + it('caches getConfig across calls', async () => { + const calls = mockApi({ + 'GET /embed/config': { body: { frameOrigin: 'https://app.monad.com/embed', apiBase: 'x' } } + }); + const monad = createEmbedClient(); + await monad.getConfig(); + await monad.getConfig(); + expect(calls.filter((c) => c.path === '/embed/config')).toHaveLength(1); + }); + + it('iconUrl derives the icon endpoint from the frame origin', async () => { + mockApi({ + 'GET /embed/config': { + body: { frameOrigin: 'https://app.monad.com/embed', apiBase: 'x' } + } + }); + const url = await createEmbedClient().iconUrl('aws-cloudtrail'); + expect(url).toBe('https://app.monad.com/external/icons/raw/aws-cloudtrail.svg'); + }); + + it('throws EmbedRequestError carrying the contract error model', async () => { + mockApi({ + 'POST /embed/pipelines/ingress': { + status: 502, + body: { code: 'upstream_error', message: 'Monad API returned 503' } + } + }); + await expect(createEmbedClient().buildIngress('in_1', 'x')).rejects.toMatchObject({ + name: 'EmbedRequestError', + status: 502, + code: 'upstream_error' + }); + expect(EmbedRequestError).toBeDefined(); + }); + + describe('openConnectorForm', () => { + function mockConfigAndSession() { + return mockApi({ + 'GET /embed/config': { + body: { frameOrigin: 'https://app.monad.com/embed', apiBase: 'https://app.monad.com/api' } + }, + 'POST /embed/session': { + body: { sessionToken: 'tok', organizationId: 'org_1', expiresAt: '2026-01-01T00:00:00Z' } + } + }); + } + + it('mints a session, resolves the frame origin, and mounts the form', async () => { + mockConfigAndSession(); + await createEmbedClient().openConnectorForm({ + container: '#modal', + kind: 'input', + typeId: 'aws-cloudtrail' + }); + expect(createConnectorFrame).toHaveBeenCalledWith( + expect.objectContaining({ + container: '#modal', + frameOrigin: 'https://app.monad.com/embed', + apiBase: 'https://app.monad.com/api', + sessionToken: 'tok', + organizationId: 'org_1', + kind: 'input', + typeId: 'aws-cloudtrail' + }) + ); + }); + + it('auto-destroys on save and forwards the connector to onSave', async () => { + mockConfigAndSession(); + const onSave = vi.fn(); + await createEmbedClient().openConnectorForm({ + container: '#modal', + kind: 'input', + typeId: 'aws-cloudtrail', + onSave + }); + const passed = createConnectorFrame.mock.calls[0]![0]; + passed.onSave!({ id: 'in_1', name: 'X' }); + expect(destroy).toHaveBeenCalledOnce(); + expect(onSave).toHaveBeenCalledWith({ id: 'in_1', name: 'X' }); + }); + + it('leaves teardown to the caller when autoDestroy is false', async () => { + mockConfigAndSession(); + await createEmbedClient().openConnectorForm({ + container: '#modal', + kind: 'input', + typeId: 'aws-cloudtrail', + autoDestroy: false + }); + createConnectorFrame.mock.calls[0]![0].onSave!({ id: 'in_1' }); + expect(destroy).not.toHaveBeenCalled(); + }); + + it('autoBuild builds the pipeline on save and fires onConnected', async () => { + mockApi({ + 'GET /embed/config': { + body: { frameOrigin: 'https://app.monad.com/embed', apiBase: 'https://app.monad.com/api' } + }, + 'POST /embed/session': { + body: { sessionToken: 'tok', organizationId: 'org_1', expiresAt: 'x' } + }, + 'POST /embed/pipelines/ingress': { + status: 201, + body: { pipelineId: 'p1', outputId: 'out_store', status: 'Running', active: true } + } + }); + const onConnected = vi.fn(); + await createEmbedClient().openConnectorForm({ + container: '#modal', + kind: 'input', + typeId: 'aws-cloudtrail', + autoBuild: true, + onConnected + }); + createConnectorFrame.mock.calls[0]![0].onSave!({ id: 'in_1', name: 'CT' }); + await vi.waitFor(() => expect(onConnected).toHaveBeenCalled()); + expect(onConnected).toHaveBeenCalledWith( + expect.objectContaining({ pipelineId: 'p1', active: true }) + ); + }); + + it('autoBuild routes a build failure to onBuildError with a working retry', async () => { + const resp = (status: number, body: unknown) => + ({ + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => JSON.stringify(body) + }) as Response; + + let ingressCalls = 0; + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string, init?: RequestInit) => { + const method = (init?.method ?? 'GET').toUpperCase(); + if (url === '/embed/config') + return resp(200, { frameOrigin: 'https://app.monad.com/embed', apiBase: 'x' }); + if (method === 'POST' && url === '/embed/session') + return resp(200, { sessionToken: 'tok', organizationId: 'org_1', expiresAt: 'x' }); + if (method === 'POST' && url === '/embed/pipelines/ingress') { + ingressCalls += 1; + return ingressCalls === 1 + ? resp(502, { code: 'upstream_error', message: 'boom' }) + : resp(201, { pipelineId: 'p1', outputId: 'o', status: 'Running', active: true }); + } + throw new Error(`unmocked ${method} ${url}`); + }) + ); + + const onConnected = vi.fn(); + const onBuildError = vi.fn(); + await createEmbedClient().openConnectorForm({ + container: '#modal', + kind: 'input', + typeId: 'aws-cloudtrail', + autoBuild: true, + onConnected, + onBuildError + }); + createConnectorFrame.mock.calls[0]![0].onSave!({ id: 'in_1', name: 'CT' }); + + await vi.waitFor(() => expect(onBuildError).toHaveBeenCalled()); + const [err, retry] = onBuildError.mock.calls[0]!; + expect(err).toMatchObject({ name: 'EmbedRequestError', status: 502, code: 'upstream_error' }); + expect(onConnected).not.toHaveBeenCalled(); + + await retry(); + expect(onConnected).toHaveBeenCalledWith(expect.objectContaining({ pipelineId: 'p1' })); + }); + }); +}); diff --git a/packages/embed/test/kit.test.ts b/packages/embed/test/kit.test.ts new file mode 100644 index 0000000..431fd91 --- /dev/null +++ b/packages/embed/test/kit.test.ts @@ -0,0 +1,140 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createEmbedClient } from '../src/kit'; + +const CFG = { apiKey: 'test-key', apiBase: 'https://api.test/api' }; + +/** Route mocked fetch by "METHOD /path" → a JSON body (or a function of the request). */ +function mockApi(routes: Record unknown)>) { + const calls: { method: string; path: string; body?: any }[] = []; + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + const method = (init?.method ?? 'GET').toUpperCase(); + const path = url.replace(CFG.apiBase, ''); + const body = init?.body ? JSON.parse(init.body as string) : undefined; + calls.push({ method, path, body }); + // Match on "METHOD /exact/path", falling back to a path prefix match so + // query strings and ids don't have to be spelled out in every route key. + const key = + Object.keys(routes).find((k) => k === `${method} ${path}`) ?? + Object.keys(routes).find((k) => { + const [m, p] = k.split(' '); + return m === method && p !== undefined && path.startsWith(p); + }); + if (!key) throw new Error(`unmocked ${method} ${path}`); + const val = routes[key]; + const resolved = typeof val === 'function' ? (val as (b: any) => unknown)(body) : val; + return { + ok: true, + status: 200, + text: async () => (resolved === undefined ? '' : JSON.stringify(resolved)) + } as Response; + }); + vi.stubGlobal('fetch', fetchMock); + return calls; +} + +afterEach(() => vi.unstubAllGlobals()); + +describe('createEmbedClient', () => { + it('mints a session and remaps snake_case → the camelCase the iframe wants', async () => { + mockApi({ + 'POST /v3/sessions': { session_token: 'tok_abc', expires_at: '2026-01-01T00:00:00Z' } + }); + const monad = createEmbedClient(CFG); + const session = await monad.team('org_1').mintSession(900); + expect(session).toEqual({ + sessionToken: 'tok_abc', + organizationId: 'org_1', + expiresAt: '2026-01-01T00:00:00Z' + }); + }); + + it('passes ttl + org through to the session mint', async () => { + const calls = mockApi({ 'POST /v3/sessions': { session_token: 't', expires_at: 'x' } }); + await createEmbedClient(CFG).mintSession('org_9', 600); + expect(calls[0]?.body).toEqual({ ttl_seconds: 600, organization_id: 'org_9' }); + }); + + it('connectSource wires input → your store when toOutputId is given', async () => { + const calls = mockApi({ + 'POST /v2/org_1/pipelines/': { id: 'pipe_1' }, + 'GET /v2/org_1/pipelines/pipe_1/status': { status: 'Running' } + }); + const built = await createEmbedClient(CFG) + .team('org_1') + .connectSource({ inputId: 'in_1', name: 'CloudTrail', toOutputId: 'out_store' }); + + expect(built).toEqual({ + pipelineId: 'pipe_1', + outputId: 'out_store', + status: 'Running', + active: true + }); + const post = calls.find((c) => c.method === 'POST' && c.path === '/v2/org_1/pipelines/'); + const nodes = post!.body.nodes; + expect(nodes).toContainEqual( + expect.objectContaining({ component_id: 'in_1', component_type: 'input' }) + ); + expect(nodes).toContainEqual( + expect.objectContaining({ component_id: 'out_store', component_type: 'output' }) + ); + }); + + it('connectSource creates a dev/null sink when no store is given', async () => { + const calls = mockApi({ + 'POST /v2/org_1/outputs': { id: 'out_devnull' }, + 'POST /v2/org_1/pipelines/': { id: 'pipe_2' }, + 'GET /v2/org_1/pipelines/pipe_2/status': { status: 'Running' } + }); + const built = await createEmbedClient(CFG) + .team('org_1') + .connectSource({ inputId: 'in_1', name: 'CloudTrail' }); + + expect(built.outputId).toBe('out_devnull'); + const outPost = calls.find((c) => c.path === '/v2/org_1/outputs'); + expect(outPost!.body.output_type).toBe('dev-null'); + }); + + it('connectDestination wires your source → the user output (egress)', async () => { + const calls = mockApi({ + 'POST /v2/org_1/pipelines/': { id: 'pipe_3' }, + 'GET /v2/org_1/pipelines/pipe_3/status': { status: 'Running' } + }); + await createEmbedClient(CFG) + .team('org_1') + .connectDestination({ outputId: 'out_user', name: 'Egress', fromInputId: 'in_source' }); + + const post = calls.find((c) => c.method === 'POST' && c.path === '/v2/org_1/pipelines/'); + const nodes = post!.body.nodes; + expect(nodes).toContainEqual( + expect.objectContaining({ component_id: 'in_source', component_type: 'input' }) + ); + expect(nodes).toContainEqual( + expect.objectContaining({ component_id: 'out_user', component_type: 'output' }) + ); + }); + + it('listConnectors normalizes the API `type` field to `type_id`', async () => { + mockApi({ + 'GET /v1/org_1/inputs': { + inputs: [{ id: 'in_1', type: 'aws-cloudtrail', name: 'Audit Logs' }] + } + }); + const rows = await createEmbedClient(CFG).team('org_1').listConnectors('input'); + expect(rows).toEqual([{ id: 'in_1', type_id: 'aws-cloudtrail', name: 'Audit Logs' }]); + }); + + it('listCatalog filters to the host allow-list', async () => { + mockApi({ + 'GET /v1/inputs': [ + { type_id: 'aws-cloudtrail', name: 'AWS CloudTrail' }, + { type_id: 'okta-systemlog', name: 'Okta' }, + { type_id: 'secret-thing', name: 'Hidden' } + ] + }); + const rows = await createEmbedClient(CFG).listCatalog('input', [ + 'aws-cloudtrail', + 'okta-systemlog' + ]); + expect(rows.map((r) => r.type_id)).toEqual(['aws-cloudtrail', 'okta-systemlog']); + }); +}); diff --git a/packages/embed/tsconfig.json b/packages/embed/tsconfig.json index 2ac349a..0dc8919 100644 --- a/packages/embed/tsconfig.json +++ b/packages/embed/tsconfig.json @@ -1,9 +1,8 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "outDir": "dist", - "rootDir": "src" + "outDir": "dist" }, - "include": ["src/**/*"], - "exclude": ["dist", "test", "node_modules"] + "include": ["src/**/*", "test/**/*"], + "exclude": ["dist", "node_modules"] } diff --git a/packages/embed/tsup.config.ts b/packages/embed/tsup.config.ts index 6bc4ba3..6717844 100644 --- a/packages/embed/tsup.config.ts +++ b/packages/embed/tsup.config.ts @@ -5,7 +5,11 @@ import { defineConfig } from 'tsup'; // map points "./connect" at these files. Add sibling entries here as // new subpath modules land (e.g. a future top-level "." aggregate). export default defineConfig({ - entry: { 'connect/index': 'src/connect/index.ts' }, + entry: { + 'connect/index': 'src/connect/index.ts', + 'kit/index': 'src/kit/index.ts', + 'client/index': 'src/client/index.ts' + }, format: ['esm', 'cjs'], dts: true, sourcemap: true, From 2f590323657cd183c498db638c25e0827abb56d0 Mon Sep 17 00:00:00 2001 From: clippy Date: Fri, 31 Jul 2026 14:10:49 -0700 Subject: [PATCH 2/7] feat(embed): add the TypeScript router + conformance suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds @monad-inc/embed-server — a zero-runtime-dependency /embed router that mounts into Express or bare Node http via createEmbedRouter (core logic + node adapter + Monad API adapter) — and the language-agnostic conformance suite that proves any backend satisfies the contract over HTTP: - conformance/ — Schemathesis response-conformance across all 9 routes plus scripted lifecycle scenarios (mint -> build -> status -> disable -> remove, and egress), driven against a stateful in-memory mock Monad. The harness boots the router under test as a subprocess (ROUTER=ts|go|python); only the ts server ships here — go/python servers land with their router layers. - A CI "Conformance (ts)" job builds the router and runs ROUTER=ts. From here, conformance is the required gate every later router PR must pass. Also ignores the Python harness .venv in prettier and formats two source files that weren't prettier-clean. Local: TS router 13 tests + conformance 14 checks green (ROUTER=ts); full typecheck / lint / format / build clean. Layer 3 (PRO-402), stacked on the browser client (PRO-401). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_011ASKA2VrufrNLGGrcFjY7n --- .github/workflows/ci.yml | 32 +++ .prettierignore | 7 +- conformance/.gitignore | 4 + conformance/README.md | 57 +++++ conformance/conftest.py | 75 +++++++ conformance/mock_monad.py | 275 +++++++++++++++++++++++ conformance/run.sh | 28 +++ conformance/servers/ts_server.mjs | 20 ++ conformance/test_conformance.py | 42 ++++ conformance/test_scenarios.py | 120 ++++++++++ pnpm-lock.yaml | 15 ++ pnpm-workspace.yaml | 1 + routers/typescript/LICENSE | 201 +++++++++++++++++ routers/typescript/README.md | 163 ++++++++++++++ routers/typescript/package.json | 58 +++++ routers/typescript/src/core.ts | 286 +++++++++++++++++++++++ routers/typescript/src/index.ts | 39 ++++ routers/typescript/src/monad.ts | 300 +++++++++++++++++++++++++ routers/typescript/src/node.ts | 96 ++++++++ routers/typescript/test/router.test.ts | 204 +++++++++++++++++ routers/typescript/tsconfig.json | 17 ++ routers/typescript/tsup.config.ts | 9 + routers/typescript/vitest.config.ts | 8 + 23 files changed, 2056 insertions(+), 1 deletion(-) create mode 100644 conformance/.gitignore create mode 100644 conformance/README.md create mode 100644 conformance/conftest.py create mode 100644 conformance/mock_monad.py create mode 100755 conformance/run.sh create mode 100644 conformance/servers/ts_server.mjs create mode 100644 conformance/test_conformance.py create mode 100644 conformance/test_scenarios.py create mode 100644 routers/typescript/LICENSE create mode 100644 routers/typescript/README.md create mode 100644 routers/typescript/package.json create mode 100644 routers/typescript/src/core.ts create mode 100644 routers/typescript/src/index.ts create mode 100644 routers/typescript/src/monad.ts create mode 100644 routers/typescript/src/node.ts create mode 100644 routers/typescript/test/router.test.ts create mode 100644 routers/typescript/tsconfig.json create mode 100644 routers/typescript/tsup.config.ts create mode 100644 routers/typescript/vitest.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6182646..d2190c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,3 +44,35 @@ jobs: - run: pnpm run typecheck - run: pnpm run test - run: pnpm run build + + conformance: + name: Conformance (ts) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - name: Build the TS router + run: pnpm -C routers/typescript build + + - uses: astral-sh/setup-uv@v5 + + - name: Set up the conformance harness + working-directory: conformance + run: | + uv venv --python 3.10 .venv + uv pip install --python .venv/bin/python schemathesis pytest httpx + + - name: Run conformance against the TS router + working-directory: conformance + env: + ROUTER: ts + run: .venv/bin/python -m pytest -q diff --git a/.prettierignore b/.prettierignore index eae7d74..edd90d5 100644 --- a/.prettierignore +++ b/.prettierignore @@ -5,4 +5,9 @@ pnpm-lock.yaml CHANGELOG.md LICENSE NOTICE -.github/CODEOWNERS \ No newline at end of file +.github/CODEOWNERS + +# Python harness tooling (conformance venv + caches) — not ours to format +**/.venv +**/.pytest_cache +**/__pycache__ \ No newline at end of file diff --git a/conformance/.gitignore b/conformance/.gitignore new file mode 100644 index 0000000..7c04452 --- /dev/null +++ b/conformance/.gitignore @@ -0,0 +1,4 @@ +.venv/ +__pycache__/ +.pytest_cache/ +.hypothesis/ diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..4e2a19f --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,57 @@ +# `/embed` conformance suite + +Proves that every backend router behaves identically and matches the +[`/embed` contract](../packages/embed/openapi/embed.openapi.yaml). It is +**language-agnostic** — it boots a router under test and drives it over HTTP, so +the same suite runs against the TypeScript, Go, and Python routers (and any +future one). + +## How it works + +``` +mock_monad.py ── a stateful in-memory stand-in for the Monad API + ▲ + │ MONAD_API_BASE +router under test ── booted by conftest.py (ROUTER=python|go|ts), mounted at /embed + ▲ + │ HTTP +tests ── test_conformance.py (Schemathesis) + test_scenarios.py (lifecycle) +``` + +- **`test_conformance.py`** — Schemathesis reads the OpenAPI spec, generates + requests for every operation, and asserts each response matches the documented + **status code, response schema, and content type**. +- **`test_scenarios.py`** — the stateful lifecycle Schemathesis's stateless + fuzzing can't cover: mint → build ingress → status → disable → status → + remove → status, plus egress. + +The router is booted with a stubbed `getCustomerOrgID` (always the tenant +`org_conf`) and provisioning (`destinationOutputId=out_store`, `sourceInputId=in_source`), +pointed at the mock. Nothing touches real Monad. + +## Run + +```sh +# one-time: create the harness venv (installs Schemathesis + the Python router) +uv venv --python 3.10 .venv +uv pip install --python .venv/bin/python schemathesis pytest httpx fastapi uvicorn -e ../routers/python + +./run.sh # all routers: python go ts +./run.sh python go # a subset +ROUTER=ts .venv/bin/python -m pytest -q # a single router directly +``` + +Prerequisites per router: **python** — none (uses the harness venv); **go** — +a Go toolchain (`go run ./cmd/conformance`); **ts** — the built package +(`run.sh` runs `pnpm -C packages/embed build` automatically). + +## Scope note — auth and negative input + +The suite runs Schemathesis's **response-conformance** checks +(`status_code_conformance`, `response_schema_conformance`, +`content_type_conformance`). It deliberately does **not** run the +`ignored_auth` or `negative_data_rejection` checks: auth is the host's +responsibility — each router mounts _behind_ the host's auth middleware and +trusts `getCustomerOrgID`, which the harness stubs — so those checks would test the +stub, not the router. A `5xx` is a documented, conformant outcome here +(`500 internal_error`, `502 upstream_error`). diff --git a/conformance/conftest.py b/conformance/conftest.py new file mode 100644 index 0000000..4265314 --- /dev/null +++ b/conformance/conftest.py @@ -0,0 +1,75 @@ +"""Conformance harness fixtures. + +Boots the stateful mock Monad plus the router under test (selected by the +``ROUTER`` env var: ``python`` | ``go`` | ``ts``), pointed at the mock, then +yields for the tests. Language-agnostic: every test drives the router over HTTP. +""" + +from __future__ import annotations + +import os +import signal +import subprocess +import time +import urllib.request + +import pytest + +import mock_monad + +ROUTER = os.environ.get("ROUTER", "python") +MOCK_PORT = int(os.environ.get("MOCK_PORT", "8790")) +ROUTER_PORT = int(os.environ.get("ROUTER_PORT", "8791")) + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO = os.path.abspath(os.path.join(_HERE, "..")) + + +def _command() -> tuple[list[str], str, dict[str, str]]: + env = dict(os.environ) + env["MONAD_API_BASE"] = f"http://127.0.0.1:{MOCK_PORT}" + env["PORT"] = str(ROUTER_PORT) + if ROUTER == "python": + return [os.path.join(_HERE, ".venv", "bin", "python"), os.path.join(_HERE, "servers", "py_server.py")], _HERE, env + if ROUTER == "go": + return ["go", "run", "./cmd/conformance"], os.path.join(_REPO, "routers", "go"), env + if ROUTER == "ts": + return ["node", os.path.join(_HERE, "servers", "ts_server.mjs")], _HERE, env + raise ValueError(f"unknown ROUTER={ROUTER!r} (expected python|go|ts)") + + +def _wait_healthy(url: str, timeout: float = 45.0) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + try: + with urllib.request.urlopen(url, timeout=1) as resp: + if resp.status < 500: + return True + except Exception: # noqa: BLE001 — server not up yet + time.sleep(0.15) + return False + + +@pytest.fixture(scope="session", autouse=True) +def servers(): + server, _state = mock_monad.start(MOCK_PORT) + cmd, cwd, env = _command() + # New session so we can kill the whole group (e.g. `go run` + its child binary). + proc = subprocess.Popen(cmd, cwd=cwd, env=env, start_new_session=True) + try: + if not _wait_healthy(f"http://127.0.0.1:{ROUTER_PORT}/embed/config"): + raise RuntimeError(f"router '{ROUTER}' did not become healthy on port {ROUTER_PORT}") + yield + finally: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + except ProcessLookupError: + pass + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except ProcessLookupError: + pass + server.shutdown() diff --git a/conformance/mock_monad.py b/conformance/mock_monad.py new file mode 100644 index 0000000..2fec298 --- /dev/null +++ b/conformance/mock_monad.py @@ -0,0 +1,275 @@ +"""A stateful in-memory stand-in for the Monad API. + +The conformance harness boots a router under test pointed at this mock (via +``MONAD_API_BASE``) so the routers run their real logic without touching real +Monad. It is stateful enough for the lifecycle scenario: creating a pipeline +stores it, listing/detail return it, PATCH updates it, DELETE removes it. + +Response shapes here mirror the **real Monad API** (from its OpenAPI / +``pkg/routes``) so a router that mis-parses a real response is caught: + +- ``GET /v1/{kind}s`` (catalog) → a bare array of connector types. +- ``GET /v1/{org}/{kind}s`` → ``{ "s": [...] | null, "pagination": {...} }`` + (the list is **null**, not ``[]``, when empty). +- ``POST /v2/{org}/pipelines`` → **201** with the full pipeline object. +- ``GET /v2/{org}/pipelines`` → ``{ "pipelines": [...], "pagination": {...} }``. +- ``GET /v2/{org}/pipelines/{id}`` → the full pipeline object at top level + (``nodes``/``edges``/``enabled`` — no ``config`` wrapper). +- ``POST /v2/{org}/outputs`` → the created output object. +- ``POST /v3/sessions`` → ``{ "session_token", "expires_at" }``. +""" + +from __future__ import annotations + +import json +import re +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + +class _State: + def __init__(self) -> None: + self.lock = threading.Lock() + self.pipelines: dict[str, dict] = {} + self._counter = 0 + + def new_id(self, prefix: str) -> str: + with self.lock: + self._counter += 1 + return f"{prefix}_{self._counter}" + + +def _component_ids(pipeline: dict) -> frozenset: + """The set of component ids a pipeline wires — used to detect a duplicate + connection (Monad rejects connecting the same components twice → 409).""" + return frozenset( + n.get("component_id") for n in pipeline.get("nodes", []) if n.get("component_id") + ) + + +def _input_id(pipeline: dict): + return next( + (n.get("component_id") for n in pipeline.get("nodes", []) if n.get("component_type") == "input"), + None, + ) + + +def _pipeline_view(p: dict) -> dict: + """A pipeline shaped like real Monad's create / detail / patch response: + the record at top level, with `nodes`/`edges`/`enabled` (no `config` wrapper).""" + return { + "id": p.get("id"), + "name": p.get("name", ""), + "description": p.get("description", ""), + "enabled": bool(p.get("enabled")), + "organization_id": "org_conf", + "managed_by": "", + "nodes": p.get("nodes", []), + "edges": p.get("edges", []), + "status": {"pipeline_id": p.get("id"), "status": "Running"}, + "createdAt": "2026-01-01T00:00:00Z", + "updatedAt": "2026-01-01T00:00:00Z", + } + + +def _pipeline_summary(p: dict) -> dict: + """A pipeline as it appears in the list endpoint (a lighter projection).""" + return { + "id": p.get("id"), + "name": p.get("name", ""), + "enabled": bool(p.get("enabled")), + "input_id": _input_id(p), + "organization_id": "org_conf", + } + + +def _output_view(oid: str, body: dict) -> dict: + return { + "id": oid, + "name": body.get("name", ""), + "description": body.get("description", ""), + "type": body.get("output_type", "dev-null"), + "organization_id": "org_conf", + "managed_by": "", + "config": {"settings": {}, "secrets": {}}, + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:00Z", + } + + +def _make_handler(state: _State): + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_args) -> None: # silence per-request logging + pass + + def _send(self, code: int, obj=None) -> None: + body = b"" if obj is None else json.dumps(obj).encode() + self.send_response(code) + if obj is not None: + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if body: + self.wfile.write(body) + + def _read(self) -> dict: + n = int(self.headers.get("Content-Length") or 0) + if n == 0: + return {} + try: + return json.loads(self.rfile.read(n) or b"{}") + except (ValueError, TypeError): + return {} + + def do_GET(self): # noqa: N802 + self._route("GET") + + def do_POST(self): # noqa: N802 + self._route("POST") + + def do_PATCH(self): # noqa: N802 + self._route("PATCH") + + def do_DELETE(self): # noqa: N802 + self._route("DELETE") + + def _route(self, method: str) -> None: + path = self.path.split("?", 1)[0] + body = self._read() if method in ("POST", "PATCH") else {} + + # Embed session mint: { session_token, expires_at }. + if method == "POST" and path == "/v3/sessions": + return self._send( + 200, {"session_token": "tok_conf", "expires_at": "2026-12-31T00:00:00Z"} + ) + + # Catalog of connector types — a bare array (no wrapper). + if method == "GET" and re.fullmatch(r"/v1/(inputs|outputs)", path): + return self._send( + 200, + [ + { + "type_id": "aws-cloudtrail", + "name": "AWS CloudTrail", + "description": "AWS CloudTrail logs", + "category": "cloud", + "in_beta": False, + }, + { + "type_id": "okta-systemlog", + "name": "Okta System Log", + "description": "Okta system log", + "category": "identity", + "in_beta": False, + }, + ], + ) + + # A tenant's configured connectors — wrapped alongside `pagination`, + # and returned as null (not []) when the tenant has none. Exercise + # both shapes: a populated `inputs`, an empty (null) `outputs`. + m = re.fullmatch(r"/v1/([^/]+)/(inputs|outputs)", path) + if method == "GET" and m: + kind = m.group(2) + rows = ( + None + if kind == "outputs" + else [ + { + "id": "cfg_1", + "type": "aws-cloudtrail", + "name": "Configured", + "organization_id": "org_conf", + "config": {"settings": {}, "secrets": {}}, + } + ] + ) + return self._send( + 200, + { + kind: rows, + "pagination": {"limit": 1000, "offset": 0, "total": 0 if rows is None else 1}, + }, + ) + + # Create an output (e.g. the dev/null sink) — the created record. + if method == "POST" and re.fullmatch(r"/v2/([^/]+)/outputs", path): + return self._send(200, _output_view(state.new_id("out"), body)) + + # pipelines collection (create / list) — check before status/detail + if re.fullmatch(r"/v2/([^/]+)/pipelines/?", path): + if method == "POST": + pid = state.new_id("pipe") + with state.lock: + new_ids = _component_ids(body) + if new_ids and any( + _component_ids(p) == new_ids for p in state.pipelines.values() + ): + return self._send( + 409, + {"error": "conflict", "message": "connector already connected"}, + ) + state.pipelines[pid] = {**body, "id": pid} + view = _pipeline_view(state.pipelines[pid]) + # Real Monad returns 201 with the full pipeline record. + return self._send(201, view) + if method == "GET": + with state.lock: + items = [_pipeline_summary(p) for p in state.pipelines.values()] + return self._send( + 200, + { + "pipelines": items, + "pagination": {"limit": 1000, "offset": 0, "total": len(items)}, + }, + ) + + if method == "GET" and re.fullmatch(r"/v2/([^/]+)/pipelines/([^/]+)/status", path): + pid = path.rsplit("/", 2)[-2] + return self._send( + 200, + { + "pipeline_id": pid, + "status": "Running", + "ingress": {"bytes": 0, "records": 0}, + "egress": {"bytes": 0, "records": 0}, + "nodes": [], + }, + ) + + m = re.fullmatch(r"/v2/([^/]+)/pipelines/([^/]+)", path) + if m: + pid = m.group(2) + if method == "GET": + with state.lock: + p = state.pipelines.get(pid) + if p is None: + return self._send(404, {"error": "pipeline not found"}) + # The full pipeline record at top level (no `config` wrapper). + return self._send(200, _pipeline_view(p)) + if method == "PATCH": + with state.lock: + if pid not in state.pipelines: + return self._send(404, {"error": "pipeline not found"}) + state.pipelines[pid] = {**state.pipelines[pid], **body, "id": pid} + view = _pipeline_view(state.pipelines[pid]) + return self._send(200, view) + if method == "DELETE": + with state.lock: + state.pipelines.pop(pid, None) + return self._send(204) + + if method == "DELETE" and re.fullmatch(r"/v1/([^/]+)/(inputs|outputs)/([^/]+)", path): + return self._send(204) + + return self._send(404, {"error": f"mock unhandled {method} {path}"}) + + return Handler + + +def start(port: int): + """Start the mock on 127.0.0.1:port in a background thread. Returns (server, state).""" + state = _State() + server = ThreadingHTTPServer(("127.0.0.1", port), _make_handler(state)) + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, state diff --git a/conformance/run.sh b/conformance/run.sh new file mode 100755 index 0000000..322f54b --- /dev/null +++ b/conformance/run.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Run the conformance suite against one or more routers. +# ./run.sh # all: python go ts +# ./run.sh python go # a subset +set -uo pipefail +cd "$(dirname "$0")" + +PY=.venv/bin/python +ROUTERS="${*:-python go ts}" + +# The TS router runs from its built standalone package. +if [[ " $ROUTERS " == *" ts "* ]]; then + echo "=== building @monad-inc/embed-server (for ts router) ===" + (cd ../routers/typescript && pnpm build >/dev/null) +fi + +rc=0 +for r in $ROUTERS; do + echo "" + echo "======================== conformance: $r ========================" + if ROUTER="$r" "$PY" -m pytest -q; then + echo "--- $r: PASS ---" + else + echo "--- $r: FAIL ---" + rc=1 + fi +done +exit $rc diff --git a/conformance/servers/ts_server.mjs b/conformance/servers/ts_server.mjs new file mode 100644 index 0000000..2a25635 --- /dev/null +++ b/conformance/servers/ts_server.mjs @@ -0,0 +1,20 @@ +// Boot the TypeScript router for conformance testing, pointed at the mock +// Monad. Imports the built standalone package (run `pnpm -C routers/typescript +// build` first). +import http from 'node:http'; +import { createEmbedRouter } from '../../routers/typescript/dist/index.js'; + +const router = createEmbedRouter({ + apiKey: 'conf-key', + apiBase: process.env.MONAD_API_BASE, + frameOrigin: 'https://app.monad.com/embed', + getCustomerOrgID: () => 'org_conf', + getProvisionedComponents: () => ({ + destinationOutputId: 'out_store', + sourceInputId: 'in_source' + }), + catalogAllow: ['aws-cloudtrail', 'okta-systemlog'] +}); + +const port = Number(process.env.PORT || 8791); +http.createServer(router).listen(port, '127.0.0.1'); diff --git a/conformance/test_conformance.py b/conformance/test_conformance.py new file mode 100644 index 0000000..67712e2 --- /dev/null +++ b/conformance/test_conformance.py @@ -0,0 +1,42 @@ +"""Schemathesis property-based conformance: does every response match the spec? + +Schemathesis reads the OpenAPI contract, generates requests for every operation, +and asserts each response conforms — documented status code, response schema, +and content type. This is language-agnostic: it drives whichever router the +``servers`` fixture booted. +""" + +import os + +import schemathesis +from schemathesis.specs.openapi.checks import ( + content_type_conformance, + response_schema_conformance, + status_code_conformance, +) + +_SPEC = os.path.join( + os.path.dirname(__file__), "..", "packages", "embed", "openapi", "embed.openapi.yaml" +) +_BASE_URL = os.environ.get("ROUTER_BASE_URL", "http://127.0.0.1:8791") + +schema = schemathesis.openapi.from_path(_SPEC) + +# We validate RESPONSE conformance: every response the router returns for every +# operation must match the spec's documented status code, response schema, and +# content type. We deliberately do NOT run Schemathesis's security/negative +# checks (`ignored_auth`, `negative_data_rejection`): auth is the host's job — +# the router mounts *behind* the host's auth middleware, and the conformance +# harness stubs `getCustomerOrgID` to always succeed — so those checks would test +# the harness stub, not the router's contract behavior. A 5xx is also a +# documented, conformant outcome here (500/502), so `not_a_server_error` is out. +_CONFORMANCE_CHECKS = [ + status_code_conformance, + response_schema_conformance, + content_type_conformance, +] + + +@schema.parametrize() +def test_response_conforms_to_spec(case): + case.call_and_validate(base_url=_BASE_URL, checks=_CONFORMANCE_CHECKS) diff --git a/conformance/test_scenarios.py b/conformance/test_scenarios.py new file mode 100644 index 0000000..6a8fef0 --- /dev/null +++ b/conformance/test_scenarios.py @@ -0,0 +1,120 @@ +"""Stateful scenario conformance — the lifecycle Schemathesis's stateless +fuzzing won't cover: mint → build → status → disable → status → remove → status. + +Runs against whichever router the ``servers`` fixture booted, over HTTP, so it +is identical for every language. +""" + +import os + +import httpx +import pytest + +_BASE_URL = os.environ.get("ROUTER_BASE_URL", "http://127.0.0.1:8791") + + +@pytest.fixture() +def client(): + with httpx.Client(base_url=_BASE_URL, timeout=15) as c: + yield c + + +def test_config_is_public(client): + r = client.get("/embed/config") + assert r.status_code == 200 + body = r.json() + assert body["frameOrigin"] == "https://app.monad.com/embed" + assert "apiBase" in body + + +def test_ingress_lifecycle(client): + # 1) mint a session for the resolved tenant + r = client.post("/embed/session") + assert r.status_code == 200, r.text + assert r.json()["organizationId"] == "org_conf" + + # 2) the iframe returned an input id → build the ingress pipeline + r = client.post("/embed/pipelines/ingress", json={"inputId": "in_conf", "name": "CloudTrail"}) + assert r.status_code == 201, r.text + built = r.json() + pipeline_id = built["pipelineId"] + assert built["outputId"] == "out_store" # wired to the provisioned store + assert built["active"] is True + + # 3) status resolves the pipeline from the input id + r = client.get("/embed/pipelines", params={"connectorId": "in_conf", "kind": "input"}) + assert r.status_code == 200, r.text + status = r.json() + assert status["hasPipeline"] is True + assert status["pipelineId"] == pipeline_id + assert status["outputId"] == "out_store" + assert status["enabled"] is True + + # 4) disable — stops flow without deleting config + r = client.post("/embed/pipelines/state", json={"pipelineId": pipeline_id, "enabled": False}) + assert r.status_code == 204, r.text + + r = client.get("/embed/pipelines", params={"connectorId": "in_conf", "kind": "input"}) + assert r.json()["enabled"] is False + + # 5) remove — the provisioned store is kept, the pipeline + input are gone + r = client.post("/embed/pipelines/remove", json={"connectorId": "in_conf", "kind": "input"}) + assert r.status_code == 204, r.text + + r = client.get("/embed/pipelines", params={"connectorId": "in_conf", "kind": "input"}) + assert r.json()["hasPipeline"] is False + + +def test_egress_builds_from_provisioned_source(client): + # the iframe returned an output id → wire the tenant's source → it + r = client.post("/embed/pipelines/egress", json={"outputId": "out_conf", "name": "Splunk"}) + assert r.status_code == 201, r.text + assert r.json()["outputId"] == "out_conf" + + +def test_catalog_is_allow_listed(client): + r = client.get("/embed/catalog", params={"kind": "input"}) + assert r.status_code == 200 + type_ids = {t["typeId"] for t in r.json()} + assert type_ids == {"aws-cloudtrail", "okta-systemlog"} + + +# Every route that takes `kind` must reject an invalid value with the shared +# error model — not just /catalog. (Schemathesis only ever sends enum-valid +# `kind`s, so this negative case has to be asserted explicitly.) +@pytest.mark.parametrize( + "call", + [ + lambda c: c.get("/embed/catalog", params={"kind": "nope"}), + lambda c: c.get("/embed/connectors", params={"kind": "nope"}), + lambda c: c.get("/embed/pipelines", params={"connectorId": "in_x", "kind": "nope"}), + lambda c: c.post("/embed/pipelines/remove", json={"connectorId": "in_x", "kind": "nope"}), + ], + ids=["catalog", "connectors", "pipelines", "remove"], +) +def test_invalid_kind_is_rejected(client, call): + r = call(client) + assert r.status_code == 400, r.text + assert r.json()["code"] == "invalid_request" + + +def test_state_on_unknown_pipeline_is_404(client): + # Toggling a pipeline that doesn't exist for this tenant is a 404 not_found — + # the router must translate Monad's 404, not fold it into a generic 502. + r = client.post( + "/embed/pipelines/state", + json={"pipelineId": "pipe_does_not_exist", "enabled": False}, + ) + assert r.status_code == 404, r.text + assert r.json()["code"] == "not_found" + + +def test_duplicate_ingress_conflicts(client): + # Connecting the same source twice collides with existing state → 409 conflict + # (a known Monad constraint the router must surface as the contract's `conflict`). + body = {"inputId": "in_dup_conflict", "name": "Dup Conflict Test"} + first = client.post("/embed/pipelines/ingress", json=body) + assert first.status_code == 201, first.text + second = client.post("/embed/pipelines/ingress", json=body) + assert second.status_code == 409, second.text + assert second.json()["code"] == "conflict" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e16ba30..fc793b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -45,6 +45,21 @@ importers: specifier: ^2.1.5 version: 2.1.9(@types/node@22.19.21)(jsdom@25.0.1) + routers/typescript: + devDependencies: + '@types/node': + specifier: ^22.9.0 + version: 22.19.21 + tsup: + specifier: ^8.3.5 + version: 8.5.1(postcss@8.5.15)(typescript@5.9.3) + typescript: + specifier: ^5.6.3 + version: 5.9.3 + vitest: + specifier: ^2.1.5 + version: 2.1.9(@types/node@22.19.21)(jsdom@25.0.1) + packages: '@asamuzakjp/css-color@3.2.0': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 18ec407..654bb7d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,2 +1,3 @@ packages: - 'packages/*' + - 'routers/typescript' diff --git a/routers/typescript/LICENSE b/routers/typescript/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/routers/typescript/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/routers/typescript/README.md b/routers/typescript/README.md new file mode 100644 index 0000000..aa713da --- /dev/null +++ b/routers/typescript/README.md @@ -0,0 +1,163 @@ +# Monad Embed — TypeScript router + +A standalone, mountable `/embed` backend router for a Monad embed integration. +Implements the [`/embed` route contract](../../packages/embed/openapi/embed.openapi.yaml). +**Zero runtime dependencies** and no dependency on any other Monad package — +lift it and go. Mounts into **Express, Fastify, Koa, and bare Node `http`** via a +Node adapter, and into **Next.js, SvelteKit, and edge runtimes** via a +framework-agnostic core. + +```ts +import express from 'express'; +import { createEmbedRouter } from '@monad-inc/embed-server'; + +const app = express(); +app.use( + '/embed', + createEmbedRouter({ + apiKey: process.env.MONAD_API_KEY!, + // your auth → the tenant's Monad team id (sync or async) + getCustomerOrgID: (req) => sessionOrg(req.raw), + // server-side lookup of a tenant's pre-provisioned components + getProvisionedComponents: (org) => ({ destinationOutputId: stores[org] }) + }) +); +``` + +Mounted with `app.use('/embed', …)`, the router serves `/embed/session`, +`/embed/pipelines/ingress`, etc. The browser holds only a session token; this +router holds the API key and is the seam to the Monad API. + +The router has two entry points, and which you use depends on your server — +**not** your frontend framework (React/Svelte are frontend; this is backend): + +- **`createEmbedRouter(cfg)`** — a Node `(req, res)` handler (it strips the + `/embed` prefix itself). Use it with **Express, Fastify, Koa, bare Node + `http`, Vite** — anything that speaks Node's req/res. +- **`createEmbedHandler(cfg)`** — the framework-agnostic core, a plain + `(EmbedRequest) => EmbedResponse` with no framework and no deps. Use it with + **Web-standard** servers — **Next.js, SvelteKit**, Hono, Bun, Deno, Cloudflare + Workers — which hand you a Fetch `Request` and take a `Response`. + +Most full-stack apps mount this in their meta-framework's own server layer +(Next.js route handlers, SvelteKit `+server.ts`), not a separate Express server. + +## Node frameworks — `createEmbedRouter` + +**Express** — [expressjs](https://expressjs.com) + +```ts +app.use('/embed', createEmbedRouter(cfg)); +``` + +**Fastify** — [fastify](https://fastify.dev) via [`@fastify/middie`](https://github.com/fastify/middie) + +```ts +import middie from '@fastify/middie'; + +await fastify.register(middie); +fastify.use('/embed', createEmbedRouter(cfg)); +``` + +**Koa** — [koajs](https://koajs.com) + +```ts +const embed = createEmbedRouter(cfg); + +app.use((ctx, next) => { + if (!ctx.path.startsWith('/embed')) return next(); + ctx.respond = false; // hand the raw res to the embed router + embed(ctx.req, ctx.res); +}); +``` + +**Bare Node `http`** — a dedicated server (routes served under `/embed`) + +```ts +import http from 'node:http'; + +http.createServer(createEmbedRouter(cfg)).listen(8080); +``` + +## Web-standard frameworks — `createEmbedHandler` + +Next.js, SvelteKit, and edge runtimes call handlers with a Fetch `Request` and +expect a `Response`. One small adapter bridges that to the core — write it once: + +```ts +import { createEmbedHandler, type EmbedRequest } from '@monad-inc/embed-server'; + +const handle = createEmbedHandler(cfg); + +// Web `Request` → `EmbedRequest` → `Response`. `subpath` is the part after +// /embed (e.g. "session", "pipelines/ingress"); `raw` is whatever your +// getCustomerOrgID reads auth from (the Request, or a framework event). +async function serve(request: Request, subpath: string, raw: unknown = request) { + const query: Record = {}; + new URL(request.url).searchParams.forEach((v, k) => (query[k] = v)); + + const res = await handle({ + method: request.method, + path: '/' + subpath, + query, + headers: Object.fromEntries(request.headers), + body: request.body ? await request.json().catch(() => undefined) : undefined, + raw + } satisfies EmbedRequest); + + return res.body === undefined + ? new Response(null, { status: res.status }) + : Response.json(res.body, { status: res.status }); +} +``` + +**Next.js** (App Router) — `app/embed/[...path]/route.ts` + +```ts +// `params` is a Promise in Next 15+; awaiting it also works on older versions. +export async function GET(request: Request, ctx: { params: Promise<{ path: string[] }> }) { + return serve(request, (await ctx.params).path.join('/')); +} +export const POST = GET; +``` + +**SvelteKit** — `src/routes/embed/[...path]/+server.ts` + +```ts +import type { RequestHandler } from './$types'; + +// Pass the whole `event` as `raw` so getCustomerOrgID can read event.locals / cookies. +const handler: RequestHandler = (event) => serve(event.request, event.params.path, event); + +export const GET = handler; +export const POST = handler; +``` + +The same adapter works on Hono, `Bun.serve`, `Deno.serve`, and Cloudflare +Workers — they all hand you a `Request` and take a `Response`. + +## Config + +| Field | Purpose | +| -------------------------- | --------------------------------------------------------------------------------------------------- | +| `apiKey` | Long-lived Monad API key (server-side only). | +| `apiBase` | Monad API base. Optional — defaults to `https://app.monad.com/api` (production). | +| `frameOrigin` | Iframe origin returned by `GET /embed/config`. Optional — defaults to production. | +| `getCustomerOrgID` | `EmbedRequest → org id`, sync or async. Return `""` / throw to reject (→ 401). | +| `getProvisionedComponents` | `org → { destinationOutputId?, sourceInputId? }`. Omit → ingress uses dev/null, egress unavailable. | +| `catalogAllow` | Restrict the catalog to these connector type ids. Omit → all. | + +## Relationship to `@monad-inc/embed` + +This router is self-contained — it inlines its own Monad client. If instead you +are writing your _own_ backend routes (not mounting a prebuilt router), the +ergonomic Monad client lives in `@monad-inc/embed/kit`. + +## Develop + +```sh +pnpm install +pnpm build # tsup → dist (ESM + CJS + d.ts) +pnpm typecheck +pnpm test # vitest +``` diff --git a/routers/typescript/package.json b/routers/typescript/package.json new file mode 100644 index 0000000..5979fc8 --- /dev/null +++ b/routers/typescript/package.json @@ -0,0 +1,58 @@ +{ + "name": "@monad-inc/embed-server", + "version": "0.1.0", + "description": "Standalone, mountable /embed backend router for a Monad embed integration. Zero runtime dependencies; mounts into Express or bare Node http.", + "license": "Apache-2.0", + "author": "Monad Inc.", + "homepage": "https://github.com/monad-inc/embed/tree/main/routers/typescript", + "repository": { + "type": "git", + "url": "git+https://github.com/monad-inc/embed.git", + "directory": "routers/typescript" + }, + "keywords": [ + "monad", + "embed", + "router", + "express", + "white-label" + ], + "type": "module", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "./package.json": "./package.json" + }, + "sideEffects": false, + "files": [ + "dist", + "README.md", + "LICENSE" + ], + "engines": { + "node": ">=18" + }, + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsup", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest" + }, + "devDependencies": { + "@types/node": "^22.9.0", + "tsup": "^8.3.5", + "typescript": "^5.6.3", + "vitest": "^2.1.5" + } +} diff --git a/routers/typescript/src/core.ts b/routers/typescript/src/core.ts new file mode 100644 index 0000000..9170968 --- /dev/null +++ b/routers/typescript/src/core.ts @@ -0,0 +1,286 @@ +/** + * @monad-inc/embed-server — the backend `/embed` router, framework-agnostic core. + * + * A standalone, self-contained implementation of the `/embed` route contract + * (see the repo's `packages/embed/openapi/embed.openapi.yaml`). It depends on + * nothing but its own inlined Monad client (`./monad`), so this package stands + * on its own — lift it and go. + * + * `createEmbedHandler` returns a plain `(EmbedRequest) => EmbedResponse` with + * ZERO runtime dependencies. A framework adapter (`./node`) turns it into a + * mountable router. + */ +import { MonadApi, UpstreamError, type ComponentKind } from './monad'; + +/** What the host pre-provisions per tenant — resolved server-side, never sent by the browser. */ +export interface Provision { + /** Ingress target: the tenant's destination output. Omit to send ingress to a dev/null sink. */ + destinationOutputId?: string; + /** Egress source: the tenant's pre-provisioned input. Required to build an egress pipeline. */ + sourceInputId?: string; +} + +/** A request normalized to the shape the core dispatches on. */ +export interface EmbedRequest { + method: string; + /** Path within the `/embed` mount, e.g. `/session`, `/pipelines/ingress`. */ + path: string; + query: Record; + headers: Record; + body?: unknown; + /** The native framework request, passed to `getCustomerOrgID` so it can read the host's auth. */ + raw?: unknown; +} + +/** A response the adapter serializes as JSON (or an empty 204). */ +export interface EmbedResponse { + status: number; + body?: unknown; +} + +/** Configuration shared by every language's router. */ +export interface EmbedServerConfig { + /** Long-lived Monad API key. Server-side only. */ + apiKey: string; + /** Monad API base. Defaults to `https://app.monad.com/api` (production); set only for non-prod. */ + apiBase?: string; + /** + * Iframe origin returned by `GET /embed/config`. Defaults to + * `https://app.monad.com/embed` (production); set only for non-prod. + */ + frameOrigin?: string; + /** + * Map the authenticated request to the caller's Monad team id. The one seam + * only the host can fill. Throw an {@link EmbedError} (or any error → 401) to + * reject an unauthenticated/unauthorized caller. + */ + getCustomerOrgID: (req: EmbedRequest) => string | Promise; + /** Per-tenant pre-provisioned resources. Omit → ingress uses dev/null and egress is unavailable. */ + getProvisionedComponents?: (org: string) => Provision | Promise; + /** Restrict the catalog to these connector type ids. Omit → expose everything. */ + catalogAllow?: string[]; +} + +/** An error carrying the HTTP status + stable code from the contract's error model. */ +export class EmbedError extends Error { + constructor( + public readonly status: number, + public readonly code: string, + message: string + ) { + super(message); + this.name = 'EmbedError'; + } +} + +const json = (status: number, body: unknown): EmbedResponse => ({ status, body }); +const noContent = (): EmbedResponse => ({ status: 204 }); + +function requireString(body: unknown, field: string): string { + const v = (body as Record | undefined)?.[field]; + if (typeof v !== 'string' || v.length === 0) { + throw new EmbedError(400, 'invalid_request', `Field '${field}' is required.`); + } + return v; +} + +function requireBoolean(body: unknown, field: string): boolean { + const v = (body as Record | undefined)?.[field]; + if (typeof v !== 'boolean') { + throw new EmbedError(400, 'invalid_request', `Field '${field}' must be a boolean.`); + } + return v; +} + +function requireKind(value: string | undefined, where: string): ComponentKind { + if (value !== 'input' && value !== 'output') { + throw new EmbedError(400, 'invalid_request', `${where} must be 'input' or 'output'.`); + } + return value; +} + +/** Wrap a Monad call so any failure surfaces as the contract's `502 upstream_error`. */ +async function upstream(p: Promise): Promise { + try { + return await p; + } catch (e) { + // Translate Monad's own status codes into the contract's error model. + if (e instanceof UpstreamError) { + if (e.status === 404) { + throw new EmbedError( + 404, + 'not_found', + 'The referenced connector or pipeline does not exist.' + ); + } + if (e.status === 409) { + throw new EmbedError(409, 'conflict', 'The request conflicts with existing state.'); + } + } + // Anything else: log the detail server-side, return a generic 502. + console.error('[embed] upstream Monad API call failed:', e); + throw new EmbedError(502, 'upstream_error', 'The upstream Monad API request failed.'); + } +} + +/** Build the framework-agnostic `/embed` handler. Hold one per process. */ +export function createEmbedHandler( + config: EmbedServerConfig +): (req: EmbedRequest) => Promise { + // Production defaults — override apiBase/frameOrigin only for non-prod. + const apiBase = config.apiBase ?? 'https://app.monad.com/api'; + const frameOrigin = config.frameOrigin ?? 'https://app.monad.com/embed'; + const monad = new MonadApi(config.apiKey, apiBase); + + async function resolve(req: EmbedRequest): Promise { + try { + const org = await config.getCustomerOrgID(req); + if (!org) { + throw new EmbedError(401, 'unauthenticated', 'Could not resolve a tenant for the request.'); + } + return org; + } catch (e) { + if (e instanceof EmbedError) throw e; + throw new EmbedError( + 401, + 'unauthenticated', + e instanceof Error ? e.message : 'Tenant resolution failed.' + ); + } + } + + async function provision(org: string): Promise { + return config.getProvisionedComponents ? await config.getProvisionedComponents(org) : {}; + } + + async function dispatch(req: EmbedRequest): Promise { + const method = req.method.toUpperCase(); + const path = req.path || '/'; + + if (method === 'GET' && path === '/config') { + return json(200, { frameOrigin, apiBase }); + } + + if (method === 'POST' && path === '/session') { + const org = await resolve(req); + return json(200, await upstream(monad.mintSession(org))); + } + + if (method === 'GET' && path === '/catalog') { + await resolve(req); + const kind = requireKind(req.query.kind, "Query 'kind'"); + return json(200, await upstream(monad.listCatalog(kind, config.catalogAllow))); + } + + if (method === 'GET' && path === '/connectors') { + const org = await resolve(req); + const kind = requireKind(req.query.kind, "Query 'kind'"); + return json(200, await upstream(monad.listConnectors(org, kind))); + } + + if (method === 'POST' && path === '/pipelines/ingress') { + const org = await resolve(req); + const inputId = requireString(req.body, 'inputId'); + const name = requireString(req.body, 'name'); + const prov = await provision(org); + const built = await upstream( + monad.connectSource(org, { inputId, name, toOutputId: prov.destinationOutputId }) + ); + return json(201, built); + } + + if (method === 'POST' && path === '/pipelines/egress') { + const org = await resolve(req); + const outputId = requireString(req.body, 'outputId'); + const name = requireString(req.body, 'name'); + const prov = await provision(org); + if (!prov.sourceInputId) { + throw new EmbedError( + 500, + 'internal_error', + 'No source input is provisioned for this tenant; egress cannot be built.' + ); + } + const built = await upstream( + monad.connectDestination(org, { outputId, name, fromInputId: prov.sourceInputId }) + ); + return json(201, built); + } + + if (method === 'GET' && path === '/pipelines') { + const org = await resolve(req); + const connectorId = req.query.connectorId; + if (!connectorId) { + throw new EmbedError(400, 'invalid_request', "Query 'connectorId' is required."); + } + const kind = requireKind(req.query.kind, "Query 'kind'"); + if (kind === 'input') { + return json(200, await upstream(monad.pipelineStatus(org, connectorId))); + } + const r = await upstream(monad.findPipelineByOutput(org, connectorId)); + return json(200, { + hasPipeline: r !== null, + enabled: r?.enabled ?? false, + pipelineId: r?.pipelineId, + inputId: r?.inputId + }); + } + + if (method === 'POST' && path === '/pipelines/state') { + const org = await resolve(req); + const pipelineId = requireString(req.body, 'pipelineId'); + const enabled = requireBoolean(req.body, 'enabled'); + await upstream(monad.setEnabled(org, pipelineId, enabled)); + return noContent(); + } + + if (method === 'POST' && path === '/pipelines/remove') { + const org = await resolve(req); + const connectorId = requireString(req.body, 'connectorId'); + const kind = requireKind( + (req.body as Record | undefined)?.kind as string | undefined, + "Field 'kind'" + ); + if (kind === 'input') { + const status = await upstream(monad.pipelineStatus(org, connectorId)); + const prov = await provision(org); + const keepStore = Boolean( + prov.destinationOutputId && + status.outputId && + prov.destinationOutputId === status.outputId + ); + await upstream( + monad.remove( + org, + { pipelineId: status.pipelineId, inputId: connectorId, outputId: status.outputId }, + { output: !keepStore } + ) + ); + } else { + const r = await upstream(monad.findPipelineByOutput(org, connectorId)); + // Keep the shared source input; remove only the pipeline + output. + await upstream( + monad.remove(org, { pipelineId: r?.pipelineId, outputId: connectorId }, { input: false }) + ); + } + return noContent(); + } + + return json(404, { code: 'not_found', message: `No route for ${method} ${path}.` }); + } + + return async (req: EmbedRequest): Promise => { + try { + return await dispatch(req); + } catch (e) { + if (e instanceof EmbedError) { + return json(e.status, { code: e.code, message: e.message }); + } + console.error('[embed] unexpected error handling request:', e); + return json(500, { + code: 'internal_error', + message: 'An unexpected error occurred.' + }); + } + }; +} diff --git a/routers/typescript/src/index.ts b/routers/typescript/src/index.ts new file mode 100644 index 0000000..0483199 --- /dev/null +++ b/routers/typescript/src/index.ts @@ -0,0 +1,39 @@ +/** + * @monad-inc/embed-server — a standalone, mountable `/embed` backend router. + * + * Implements the `/embed` route contract for a Monad embed integration. The + * host mounts this behind its own auth; the browser holds only a session token, + * this router holds the API key and is the seam to the Monad API. Depends on + * nothing but Node — lift it and go. + * + * import { createEmbedRouter } from '@monad-inc/embed-server'; + * + * app.use('/embed', createEmbedRouter({ + * apiKey: process.env.MONAD_API_KEY!, + * apiBase: 'https://app.monad.com/api', + * frameOrigin: 'https://app.monad.com/embed', + * getCustomerOrgID: (req) => sessionOrg(req.raw), // your auth → Monad team + * getProvisionedComponents: (org) => ({ destinationOutputId: stores[org] }), + * })); + */ +export { + createEmbedHandler, + EmbedError, + type EmbedServerConfig, + type EmbedRequest, + type EmbedResponse, + type Provision +} from './core'; + +export { createEmbedRouter, type EmbedRouterOptions } from './node'; + +export { + MonadApi, + type ComponentKind, + type Session, + type CatalogType, + type ConfiguredConnector, + type BuiltPipeline, + type PipelineStatus, + type CleanupPolicy +} from './monad'; diff --git a/routers/typescript/src/monad.ts b/routers/typescript/src/monad.ts new file mode 100644 index 0000000..e1bb381 --- /dev/null +++ b/routers/typescript/src/monad.ts @@ -0,0 +1,300 @@ +/** + * A self-contained Monad API client — the TypeScript equivalent of the Go and + * Python routers' inlined clients. This package stands on its own: it does NOT + * depend on `@monad-inc/embed`. It sequences Monad's /v1 + /v2 + /v3 calls and + * returns values already shaped to the `/embed` contract (camelCase). + * + * A failed Monad call throws; the router maps that to `502 upstream_error`. + */ + +export type ComponentKind = 'input' | 'output'; + +export interface Session { + sessionToken: string; + organizationId: string; + expiresAt: string; +} + +export interface CatalogType { + typeId: string; + name: string; +} + +export interface ConfiguredConnector { + id: string; + typeId: string; + name: string; +} + +export interface BuiltPipeline { + pipelineId: string; + outputId: string; + status: string; + active: boolean; +} + +export interface PipelineStatus { + hasPipeline: boolean; + enabled: boolean; + pipelineId?: string; + inputId?: string; + outputId?: string; +} + +/** Which resources a delete removes. A flag left undefined defaults to true. */ +export interface CleanupPolicy { + pipeline?: boolean; + input?: boolean; + output?: boolean; +} + +const POLL_ATTEMPTS = 15; +const POLL_INTERVAL_MS = 2000; + +/** URL-encode a single path segment so browser-supplied ids can't inject query + * params or traverse the path (`/`, `?`, `#` are neutralised). */ +const seg = (s: string): string => encodeURIComponent(s); + +/** + * Thrown when a Monad API call fails. Carries the status + internal detail for + * server-side logging; the router surfaces only the generic super() message to + * the browser, so upstream response bodies never cross the trust boundary. + */ +export class UpstreamError extends Error { + constructor( + readonly status: number, + readonly detail: string + ) { + super('The upstream Monad API request failed.'); + this.name = 'UpstreamError'; + } +} + +/* eslint-disable @typescript-eslint/no-explicit-any -- Monad responses are untyped JSON */ + +/** The Monad API client. Bind your key + base once; call plain methods. */ +export class MonadApi { + private readonly apiBase: string; + private readonly apiKey: string; + + constructor(apiKey: string, apiBase: string) { + this.apiKey = apiKey; + this.apiBase = apiBase; + } + + private async req(path: string, init?: RequestInit): Promise { + const r = await fetch(`${this.apiBase}${path}`, { + ...init, + headers: { + 'Content-Type': 'application/json', + Authorization: `ApiKey ${this.apiKey}`, + ...(init?.headers ?? {}) + } + }); + if (!r.ok) { + // Keep the upstream body server-side only — never surface it to the browser. + throw new UpstreamError(r.status, `${r.status} ${path}: ${(await r.text()).slice(0, 300)}`); + } + const text = await r.text(); + return text ? JSON.parse(text) : undefined; + } + + async mintSession(org: string): Promise { + const raw = await this.req('/v3/sessions', { + method: 'POST', + body: JSON.stringify({ ttl_seconds: 1800, organization_id: org }) + }); + return { sessionToken: raw.session_token, organizationId: org, expiresAt: raw.expires_at }; + } + + async listCatalog(kind: ComponentKind, allow?: string[]): Promise { + const types = (await this.req(`/v1/${kind}s`)) as { type_id: string; name: string }[]; + const list = types.map((t) => ({ typeId: t.type_id, name: t.name })); + if (!allow || allow.length === 0) return list; + const set = new Set(allow); + return list.filter((t) => set.has(t.typeId)); + } + + async listConnectors(org: string, kind: ComponentKind): Promise { + const page = (await this.req(`/v1/${seg(org)}/${kind}s?limit=1000&offset=0`)) as Record< + string, + { id: string; type: string; name: string }[] + >; + const rows = page?.[`${kind}s`] ?? []; + return rows.map((r) => ({ id: r.id, typeId: r.type, name: r.name })); + } + + private async wire( + org: string, + inputId: string, + outputId: string, + name: string + ): Promise { + const pipeline = (await this.req(`/v2/${seg(org)}/pipelines/`, { + method: 'POST', + body: JSON.stringify({ + name, + description: 'Created when the connector was configured via embed', + enabled: true, + nodes: [ + { slug: 'in', component_id: inputId, component_type: 'input', enabled: true }, + { slug: 'out', component_id: outputId, component_type: 'output', enabled: true } + ], + edges: [ + { + from_node_instance_id: 'in', + to_node_instance_id: 'out', + description: 'all records', + conditions: { operator: 'always' } + } + ] + }) + })) as { id: string }; + + let status = 'Pending'; + for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { + const s = (await this.req(`/v2/${seg(org)}/pipelines/${seg(pipeline.id)}/status`)) as { + status?: string; + }; + status = s?.status ?? status; + if (status === 'Running' || status === 'Erroring') break; + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + } + return { pipelineId: pipeline.id, outputId, status, active: status === 'Running' }; + } + + private async buildDevNull(org: string, inputId: string, name: string): Promise { + const output = (await this.req(`/v2/${seg(org)}/outputs`, { + method: 'POST', + body: JSON.stringify({ + output_type: 'dev-null', + name: `${name} → /dev/null`, + description: 'Auto-created sink for embed pipeline', + promise_id: '', + config: { settings: {}, secrets: {} } + }) + })) as { id: string }; + return this.wire(org, inputId, output.id, name); + } + + /** Ingress: wire a configured input → the store, or a dev/null sink if none. */ + connectSource( + org: string, + opts: { inputId: string; name: string; toOutputId?: string } + ): Promise { + if (opts.toOutputId) return this.wire(org, opts.inputId, opts.toOutputId, opts.name); + return this.buildDevNull(org, opts.inputId, opts.name); + } + + /** Egress: wire the pre-provisioned source → a configured output. */ + connectDestination( + org: string, + opts: { outputId: string; name: string; fromInputId: string } + ): Promise { + return this.wire(org, opts.fromInputId, opts.outputId, opts.name); + } + + private async pipelines(org: string): Promise { + const listed = await this.req(`/v2/${seg(org)}/pipelines/`); + return Array.isArray(listed) ? listed : (listed?.pipelines ?? listed?.data ?? []); + } + + private async detail(org: string, id: string): Promise { + const d = await this.req(`/v2/${seg(org)}/pipelines/${seg(id)}`); + return d?.config ?? d ?? {}; + } + + async pipelineStatus(org: string, inputId: string): Promise { + for (const summary of await this.pipelines(org)) { + if (!summary?.id) continue; + let p: any; + try { + p = await this.detail(org, summary.id); + } catch { + continue; + } + const nodes: any[] = p.nodes ?? []; + const inNode = nodes.find((n) => n.component_type === 'input' && n.component_id === inputId); + if (!inNode) continue; + const outNode = nodes.find((n) => n.component_type === 'output'); + return { + hasPipeline: true, + enabled: Boolean(p.enabled), + pipelineId: summary.id, + outputId: outNode?.component_id + }; + } + return { hasPipeline: false, enabled: false }; + } + + async findPipelineByOutput(org: string, outputId: string): Promise { + for (const summary of await this.pipelines(org)) { + if (!summary?.id) continue; + let p: any; + try { + p = await this.detail(org, summary.id); + } catch { + continue; + } + const nodes: any[] = p.nodes ?? []; + const outNode = nodes.find( + (n) => n.component_type === 'output' && n.component_id === outputId + ); + if (!outNode) continue; + const inNode = nodes.find((n) => n.component_type === 'input'); + return { + hasPipeline: true, + enabled: Boolean(p.enabled), + pipelineId: summary.id, + inputId: inNode?.component_id + }; + } + return null; + } + + async setEnabled(org: string, pipelineId: string, enabled: boolean): Promise { + const p = await this.detail(org, pipelineId); + await this.req(`/v2/${seg(org)}/pipelines/${seg(pipelineId)}`, { + method: 'PATCH', + body: JSON.stringify({ + name: p.name, + description: p.description ?? '', + enabled, + nodes: (p.nodes ?? []).map((n: any) => ({ + id: n.id, + slug: n.slug, + component_id: n.component_id, + component_type: n.component_type, + enabled: n.enabled ?? true + })), + edges: (p.edges ?? []).map((e: any) => ({ + name: e.name, + description: e.description ?? '', + from_node_instance_id: e.from_node_instance_id, + to_node_instance_id: e.to_node_instance_id, + disabled: e.disabled ?? false, + conditions: e.conditions + })) + }) + }); + } + + /** Delete resources per a cleanup policy. Order: pipeline (references the rest) first. */ + async remove( + org: string, + ids: { pipelineId?: string; inputId?: string; outputId?: string }, + cleanup: CleanupPolicy = {} + ): Promise { + const policy = { pipeline: true, input: true, output: true, ...cleanup }; + if (ids.pipelineId && policy.pipeline) { + await this.req(`/v2/${seg(org)}/pipelines/${seg(ids.pipelineId)}`, { method: 'DELETE' }); + } + if (ids.inputId && policy.input) { + await this.req(`/v1/${seg(org)}/inputs/${seg(ids.inputId)}`, { method: 'DELETE' }); + } + if (ids.outputId && policy.output) { + await this.req(`/v1/${seg(org)}/outputs/${seg(ids.outputId)}`, { method: 'DELETE' }); + } + } +} +/* eslint-enable @typescript-eslint/no-explicit-any */ diff --git a/routers/typescript/src/node.ts b/routers/typescript/src/node.ts new file mode 100644 index 0000000..f2a686a --- /dev/null +++ b/routers/typescript/src/node.ts @@ -0,0 +1,96 @@ +/** + * Node adapter — turns the framework-agnostic core into a mountable router. + * + * `import type` from `node:http` is erased at build time, so this keeps the + * package's zero-runtime-dependency guarantee. The returned `(req, res)` + * function works with bare Node `http` and with Express (Express's req/res + * extend Node's): + * + * app.use('/embed', createEmbedRouter(config)); // Express + * http.createServer(createEmbedRouter(config)); // bare Node (routes under /embed) + */ +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { createEmbedHandler, type EmbedRequest, type EmbedServerConfig } from './core'; + +export interface EmbedRouterOptions { + /** + * Path prefix to strip from the incoming URL before matching. Defaults to + * `/embed`. When mounted with `app.use('/embed', …)` Express has already + * stripped it; when used as a raw Node handler the full `/embed/...` path + * arrives and this removes the prefix. + */ + mountPath?: string; +} + +function headerRecord(headers: IncomingMessage['headers']): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(headers)) { + out[k] = Array.isArray(v) ? v[0] : v; + } + return out; +} + +async function readJsonBody(req: IncomingMessage): Promise { + const parsed = (req as { body?: unknown }).body; + if (parsed !== undefined) return parsed; + + const method = (req.method ?? 'GET').toUpperCase(); + if (method === 'GET' || method === 'HEAD') return undefined; + + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : (chunk as Buffer)); + } + if (chunks.length === 0) return undefined; + const text = Buffer.concat(chunks).toString('utf8').trim(); + if (!text) return undefined; + try { + return JSON.parse(text); + } catch { + return undefined; + } +} + +/** Build a `(req, res)` router that serves the `/embed` routes. */ +export function createEmbedRouter(config: EmbedServerConfig, options: EmbedRouterOptions = {}) { + const handle = createEmbedHandler(config); + const mountPath = options.mountPath ?? '/embed'; + + return function embedRouter(req: IncomingMessage, res: ServerResponse): void { + void (async () => { + const url = new URL(req.url ?? '/', 'http://embed.local'); + let path = url.pathname; + if (path === mountPath) path = '/'; + else if (path.startsWith(mountPath + '/')) path = path.slice(mountPath.length); + + const query: Record = {}; + for (const [k, v] of url.searchParams) query[k] = v; + + const embedReq: EmbedRequest = { + method: req.method ?? 'GET', + path, + query, + headers: headerRecord(req.headers), + body: await readJsonBody(req), + raw: req + }; + + const result = await handle(embedReq); + res.statusCode = result.status; + if (result.body === undefined) { + res.end(); + return; + } + res.setHeader('content-type', 'application/json; charset=utf-8'); + res.end(JSON.stringify(result.body)); + })().catch(() => { + if (!res.headersSent) { + res.statusCode = 500; + res.setHeader('content-type', 'application/json; charset=utf-8'); + res.end(JSON.stringify({ code: 'internal_error', message: 'Request handling failed.' })); + } else { + res.end(); + } + }); + }; +} diff --git a/routers/typescript/test/router.test.ts b/routers/typescript/test/router.test.ts new file mode 100644 index 0000000..a0470b7 --- /dev/null +++ b/routers/typescript/test/router.test.ts @@ -0,0 +1,204 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createEmbedHandler, type EmbedRequest, type EmbedServerConfig } from '../src'; + +const API_BASE = 'https://api.test/api'; + +/** Route mocked fetch by "METHOD /path" → a JSON body (function receives the parsed request body). */ +function mockApi(routes: Record unknown)>) { + const calls: { method: string; path: string; body?: unknown }[] = []; + const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { + const method = (init?.method ?? 'GET').toUpperCase(); + const path = url.replace(API_BASE, ''); + const body = init?.body ? JSON.parse(init.body as string) : undefined; + calls.push({ method, path, body }); + const key = + Object.keys(routes).find((k) => k === `${method} ${path}`) ?? + Object.keys(routes).find((k) => { + const [m, p] = k.split(' '); + return m === method && p !== undefined && path.startsWith(p); + }); + if (!key) throw new Error(`unmocked ${method} ${path}`); + const val = routes[key]; + const resolved = typeof val === 'function' ? (val as (b: unknown) => unknown)(body) : val; + return { + ok: true, + status: 200, + text: async () => (resolved === undefined ? '' : JSON.stringify(resolved)) + } as Response; + }); + vi.stubGlobal('fetch', fetchMock); + return calls; +} + +const baseConfig = (over: Partial = {}): EmbedServerConfig => ({ + apiKey: 'k', + apiBase: API_BASE, + frameOrigin: 'https://app.monad.com/embed', + getCustomerOrgID: () => 'org_1', + ...over +}); + +const req = (method: string, path: string, extra: Partial = {}): EmbedRequest => ({ + method, + path, + query: {}, + headers: {}, + ...extra +}); + +afterEach(() => vi.unstubAllGlobals()); + +describe('createEmbedHandler (standalone)', () => { + it('GET /config returns frame + api config without auth', async () => { + const handle = createEmbedHandler( + baseConfig({ + getCustomerOrgID: () => { + throw new Error('should not be called'); + } + }) + ); + const res = await handle(req('GET', '/config')); + expect(res).toEqual({ + status: 200, + body: { frameOrigin: 'https://app.monad.com/embed', apiBase: API_BASE } + }); + }); + + it('defaults apiBase + frameOrigin to production when omitted', async () => { + const res = await createEmbedHandler({ apiKey: 'k', getCustomerOrgID: () => 'org_1' })( + req('GET', '/config') + ); + expect(res.body).toEqual({ + frameOrigin: 'https://app.monad.com/embed', + apiBase: 'https://app.monad.com/api' + }); + }); + + it('POST /session mints a token for the resolved tenant', async () => { + mockApi({ 'POST /v3/sessions': { session_token: 'tok', expires_at: '2026-01-01T00:00:00Z' } }); + const res = await createEmbedHandler(baseConfig())(req('POST', '/session')); + expect(res.status).toBe(200); + expect(res.body).toEqual({ + sessionToken: 'tok', + organizationId: 'org_1', + expiresAt: '2026-01-01T00:00:00Z' + }); + }); + + it('GET /catalog returns the contract camelCase (typeId), filtered to the allow-list', async () => { + mockApi({ + 'GET /v1/inputs': [ + { type_id: 'aws-cloudtrail', name: 'AWS' }, + { type_id: 'secret', name: 'Hidden' } + ] + }); + const handle = createEmbedHandler(baseConfig({ catalogAllow: ['aws-cloudtrail'] })); + const res = await handle(req('GET', '/catalog', { query: { kind: 'input' } })); + expect(res.status).toBe(200); + expect(res.body).toEqual([{ typeId: 'aws-cloudtrail', name: 'AWS' }]); + }); + + it('GET /connectors normalizes the API `type` field to camelCase `typeId`', async () => { + mockApi({ + 'GET /v1/org_1/inputs': { + inputs: [{ id: 'in_1', type: 'aws-cloudtrail', name: 'Audit Logs' }] + } + }); + const res = await createEmbedHandler(baseConfig())( + req('GET', '/connectors', { query: { kind: 'input' } }) + ); + expect(res.body).toEqual([{ id: 'in_1', typeId: 'aws-cloudtrail', name: 'Audit Logs' }]); + }); + + it('POST /pipelines/ingress wires to the provisioned store', async () => { + const calls = mockApi({ + 'POST /v2/org_1/pipelines/': { id: 'pipe_1' }, + 'GET /v2/org_1/pipelines/pipe_1/status': { status: 'Running' } + }); + const handle = createEmbedHandler( + baseConfig({ getProvisionedComponents: () => ({ destinationOutputId: 'out_store' }) }) + ); + const res = await handle( + req('POST', '/pipelines/ingress', { body: { inputId: 'in_1', name: 'CT' } }) + ); + expect(res.status).toBe(201); + expect((res.body as { outputId: string }).outputId).toBe('out_store'); + const post = calls.find((c) => c.method === 'POST' && c.path === '/v2/org_1/pipelines/'); + expect((post!.body as { nodes: unknown[] }).nodes).toContainEqual( + expect.objectContaining({ component_id: 'out_store', component_type: 'output' }) + ); + }); + + it('POST /pipelines/ingress falls back to a dev/null sink without a store', async () => { + mockApi({ + 'POST /v2/org_1/outputs': { id: 'out_devnull' }, + 'POST /v2/org_1/pipelines/': { id: 'pipe_2' }, + 'GET /v2/org_1/pipelines/pipe_2/status': { status: 'Running' } + }); + const res = await createEmbedHandler(baseConfig())( + req('POST', '/pipelines/ingress', { body: { inputId: 'in_1', name: 'CT' } }) + ); + expect(res.status).toBe(201); + expect((res.body as { outputId: string }).outputId).toBe('out_devnull'); + }); + + it('POST /pipelines/egress 500s when no source is provisioned', async () => { + const res = await createEmbedHandler(baseConfig())( + req('POST', '/pipelines/egress', { body: { outputId: 'out_1', name: 'Splunk' } }) + ); + expect(res.status).toBe(500); + expect((res.body as { code: string }).code).toBe('internal_error'); + }); + + it('POST /pipelines/state disables and returns 204', async () => { + mockApi({ + 'GET /v2/org_1/pipelines/pipe_1': { + config: { name: 'p', nodes: [], edges: [], enabled: true } + }, + 'PATCH /v2/org_1/pipelines/pipe_1': undefined + }); + const res = await createEmbedHandler(baseConfig())( + req('POST', '/pipelines/state', { body: { pipelineId: 'pipe_1', enabled: false } }) + ); + expect(res).toEqual({ status: 204 }); + }); + + it('rejects an unauthenticated caller with 401', async () => { + const handle = createEmbedHandler( + baseConfig({ + getCustomerOrgID: () => { + throw new Error('no session'); + } + }) + ); + const res = await handle(req('POST', '/session')); + expect(res.status).toBe(401); + expect((res.body as { code: string }).code).toBe('unauthenticated'); + }); + + it('400s on an invalid kind and on a missing body field', async () => { + const handle = createEmbedHandler(baseConfig()); + const bad = await handle(req('GET', '/catalog', { query: { kind: 'nope' } })); + expect(bad.status).toBe(400); + const missing = await handle(req('POST', '/pipelines/ingress', { body: { name: 'x' } })); + expect(missing.status).toBe(400); + }); + + it('maps a Monad API failure to 502 upstream_error', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('connection refused'); + }) + ); + const res = await createEmbedHandler(baseConfig())(req('POST', '/session')); + expect(res.status).toBe(502); + expect((res.body as { code: string }).code).toBe('upstream_error'); + }); + + it('404s an unknown route', async () => { + const res = await createEmbedHandler(baseConfig())(req('GET', '/nope')); + expect(res.status).toBe(404); + expect((res.body as { code: string }).code).toBe('not_found'); + }); +}); diff --git a/routers/typescript/tsconfig.json b/routers/typescript/tsconfig.json new file mode 100644 index 0000000..86a8c7a --- /dev/null +++ b/routers/typescript/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "lib": ["ES2022", "DOM"], + "strict": true, + "noUncheckedIndexedAccess": true, + "esModuleInterop": true, + "skipLibCheck": true, + "declaration": true, + "outDir": "dist", + "types": ["node"] + }, + "include": ["src/**/*", "test/**/*"], + "exclude": ["dist", "node_modules"] +} diff --git a/routers/typescript/tsup.config.ts b/routers/typescript/tsup.config.ts new file mode 100644 index 0000000..d2fc6b4 --- /dev/null +++ b/routers/typescript/tsup.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig({ + entry: { index: 'src/index.ts' }, + format: ['esm', 'cjs'], + dts: true, + sourcemap: true, + clean: true +}); diff --git a/routers/typescript/vitest.config.ts b/routers/typescript/vitest.config.ts new file mode 100644 index 0000000..08d237f --- /dev/null +++ b/routers/typescript/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['test/**/*.test.ts'], + environment: 'node' + } +}); From 03b78c9fee2ddd0b6d3eba69eac33bf3bfd91cac Mon Sep 17 00:00:00 2001 From: clippy Date: Fri, 31 Jul 2026 15:42:33 -0700 Subject: [PATCH 3/7] test(conformance): live mode against real Monad + Monad-schema mock fidelity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a live conformance mode (MONAD_LIVE=1) that boots the router against the real Monad API instead of the mock, driven by env-configurable credentials/org/ provisioned ids (all defaulting to the mock fixtures, so the hermetic run is unchanged). Schemathesis is restricted to read-only GET operations in live mode. Adds test_mock_fidelity.py + monad_schemas.py, which validate every mock response against the Monad response shapes the routers consume — pinning the mock to the documented upstream contract (needs jsonschema). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_011ASKA2VrufrNLGGrcFjY7n --- .github/workflows/ci.yml | 2 +- conformance/README.md | 47 ++++++++++- conformance/conftest.py | 15 +++- conformance/monad_schemas.py | 128 ++++++++++++++++++++++++++++++ conformance/servers/ts_server.mjs | 34 +++++--- conformance/test_conformance.py | 8 ++ conformance/test_mock_fidelity.py | 102 ++++++++++++++++++++++++ conformance/test_scenarios.py | 118 ++++++++++++++++++++++----- 8 files changed, 414 insertions(+), 40 deletions(-) create mode 100644 conformance/monad_schemas.py create mode 100644 conformance/test_mock_fidelity.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2190c6..360c92d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,7 @@ jobs: working-directory: conformance run: | uv venv --python 3.10 .venv - uv pip install --python .venv/bin/python schemathesis pytest httpx + uv pip install --python .venv/bin/python schemathesis pytest httpx jsonschema - name: Run conformance against the TS router working-directory: conformance diff --git a/conformance/README.md b/conformance/README.md index 4e2a19f..00fd5d2 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -16,19 +16,60 @@ router under test ── booted by conftest.py (ROUTER=python|go|ts), mounted a ▲ │ HTTP tests ── test_conformance.py (Schemathesis) + test_scenarios.py (lifecycle) + + test_mock_fidelity.py (mock ⟷ Monad shapes) ``` -- **`test_conformance.py`** — Schemathesis reads the OpenAPI spec, generates - requests for every operation, and asserts each response matches the documented - **status code, response schema, and content type**. +- **`test_conformance.py`** — Schemathesis reads the `/embed` OpenAPI spec, + generates requests for every operation, and asserts each response matches the + documented **status code, response schema, and content type**. - **`test_scenarios.py`** — the stateful lifecycle Schemathesis's stateless fuzzing can't cover: mint → build ingress → status → disable → status → remove → status, plus egress. +- **`test_mock_fidelity.py`** — pins the mock to the **Monad** response shapes + the routers consume (`monad_schemas.py`, vendored from Monad's OpenAPI), so the + mock can't drift from the documented upstream contract on a field a router + parses. This is the check that would have caught a missing `pagination` + sibling for free. (Needs `jsonschema`.) The router is booted with a stubbed `getCustomerOrgID` (always the tenant `org_conf`) and provisioning (`destinationOutputId=out_store`, `sourceInputId=in_source`), pointed at the mock. Nothing touches real Monad. +## Two contracts, two directions + +The suite guards both contracts the router sits between: + +- **downstream** — `test_conformance.py` proves the router's `/embed` responses + match _our_ published contract (`packages/embed/openapi/embed.openapi.yaml`). +- **upstream** — `test_mock_fidelity.py` proves the _mock_ matches _Monad's_ + documented shapes. But a spec is only as good as its accuracy: Monad's spec + types the connectors list as `array` yet real Monad returns `null` when empty + (the bug that hit the Python router). So spec-fidelity alone is not enough — + **live mode** (below) closes the gap by running the routers against real Monad. + +## Live mode — run the routers against real Monad + +The default run is hermetic (mock). Set `MONAD_LIVE=1` and point the same env at +real Monad staging to run the **identical** read-path scenarios against the real +API — the only thing that catches where reality diverges from the spec: + +```sh +MONAD_LIVE=1 \ +MONAD_API_BASE=https://app.monad.security/api \ +MONAD_API_KEY= \ +MONAD_ORG_ID= \ +MONAD_SOURCE_ID= \ +MONAD_FRAME_ORIGIN=https://app.monad.security/embed \ +ROUTER=python .venv/bin/python -m pytest -q +``` + +In live mode the mock is not booted; Schemathesis is restricted to **read-only +GET** operations (no fuzzed mutations against real Monad); and the mutating +lifecycle scenarios are skipped unless you opt in with `MONAD_LIVE_MUTATE=1` +plus throwaway `CONF_INPUT_ID` / `CONF_OUTPUT_ID` components (they create and +then delete real pipelines, keeping the shared source). Every knob defaults to +the mock fixture value, so an unset env is exactly the hermetic run. + ## Run ```sh diff --git a/conformance/conftest.py b/conformance/conftest.py index 4265314..08b8900 100644 --- a/conformance/conftest.py +++ b/conformance/conftest.py @@ -21,13 +21,20 @@ MOCK_PORT = int(os.environ.get("MOCK_PORT", "8790")) ROUTER_PORT = int(os.environ.get("ROUTER_PORT", "8791")) +# Live mode boots the router against the real Monad API (MONAD_API_BASE + +# MONAD_API_KEY + MONAD_ORG_ID … supplied by the caller) instead of the mock. +LIVE = os.environ.get("MONAD_LIVE") == "1" + _HERE = os.path.dirname(os.path.abspath(__file__)) _REPO = os.path.abspath(os.path.join(_HERE, "..")) def _command() -> tuple[list[str], str, dict[str, str]]: env = dict(os.environ) - env["MONAD_API_BASE"] = f"http://127.0.0.1:{MOCK_PORT}" + # In mock mode we own MONAD_API_BASE (point the router at the local mock); + # in live mode the caller's real MONAD_API_BASE is passed through untouched. + if not LIVE: + env["MONAD_API_BASE"] = f"http://127.0.0.1:{MOCK_PORT}" env["PORT"] = str(ROUTER_PORT) if ROUTER == "python": return [os.path.join(_HERE, ".venv", "bin", "python"), os.path.join(_HERE, "servers", "py_server.py")], _HERE, env @@ -52,7 +59,8 @@ def _wait_healthy(url: str, timeout: float = 45.0) -> bool: @pytest.fixture(scope="session", autouse=True) def servers(): - server, _state = mock_monad.start(MOCK_PORT) + # Live mode talks to real Monad — no mock to boot. + server = None if LIVE else mock_monad.start(MOCK_PORT)[0] cmd, cwd, env = _command() # New session so we can kill the whole group (e.g. `go run` + its child binary). proc = subprocess.Popen(cmd, cwd=cwd, env=env, start_new_session=True) @@ -72,4 +80,5 @@ def servers(): os.killpg(os.getpgid(proc.pid), signal.SIGKILL) except ProcessLookupError: pass - server.shutdown() + if server is not None: + server.shutdown() diff --git a/conformance/monad_schemas.py b/conformance/monad_schemas.py new file mode 100644 index 0000000..5494bf1 --- /dev/null +++ b/conformance/monad_schemas.py @@ -0,0 +1,128 @@ +"""JSON Schemas for the **Monad API** responses the routers consume. + +These are the *upstream* contract the routers depend on — distinct from our own +``/embed`` contract (``packages/embed/openapi/embed.openapi.yaml``, which +Schemathesis checks). They are hand-authored from Monad's OpenAPI +(``docs/swagger.json``) + ``pkg/routes/embed`` — deliberately vendored rather +than importing Monad's internal spec into this public repo, and pared to the +fields the routers actually read. + +``test_mock_fidelity.py`` validates every response the stateful mock emits +against these, so the mock can never drift from the documented Monad shapes on a +field a router parses. This is the check that would have caught the missing +``pagination`` sibling for free. Each schema allows extra properties — real +Monad returns many more fields than the routers touch. + +Notes where the mock encodes reality *beyond* the spec: +- the connectors list field is ``["array", "null"]``: real Monad returns + ``null`` (not ``[]``) for a tenant with none, though its spec types it + ``array``. Live conformance is what proves this, but pinning it here stops the + mock from regressing to a non-null-emitting shape. +""" + +# POST /v3/sessions — the embed session mint (swagger leaves the body untyped; +# shape is from pkg/routes/embed/sessions.go). +SESSION = { + "type": "object", + "required": ["session_token", "expires_at"], + "properties": { + "session_token": {"type": "string"}, + "expires_at": {"type": "string"}, + }, +} + +# GET /v1/{kind}s — the connector-type catalog: a bare array. +CATALOG = { + "type": "array", + "items": { + "type": "object", + "required": ["type_id", "name"], + "properties": { + "type_id": {"type": "string"}, + "name": {"type": "string"}, + }, + }, +} + + +def connectors_list(kind: str) -> dict: + """GET /v1/{org}/{kind}s — the tenant's configured connectors, wrapped + alongside ``pagination``. The list is nullable (see module note).""" + return { + "type": "object", + "required": [f"{kind}s", "pagination"], + "properties": { + f"{kind}s": { + "type": ["array", "null"], + "items": { + "type": "object", + "required": ["id", "type", "name"], + "properties": { + "id": {"type": "string"}, + "type": {"type": "string"}, + "name": {"type": "string"}, + }, + }, + }, + "pagination": {"type": "object"}, + }, + } + + +# A pipeline node as the routers read it (match on component_type/component_id). +_NODE = { + "type": "object", + "required": ["component_type", "component_id"], + "properties": { + "component_type": {"type": "string"}, + "component_id": {"type": "string"}, + "slug": {"type": "string"}, + "enabled": {"type": "boolean"}, + }, +} + +# GET/POST/PATCH /v2/{org}/pipelines/{id} — the full pipeline record. The routers +# read id/name/enabled + the nodes to resolve input/output wiring. +PIPELINE = { + "type": "object", + "required": ["id", "enabled", "nodes"], + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "description": {"type": "string"}, + "enabled": {"type": "boolean"}, + "nodes": {"type": "array", "items": _NODE}, + "edges": {"type": "array"}, + }, +} + +# GET /v2/{org}/pipelines/ — the list. Router accepts a bare array or a wrapper. +PIPELINE_LIST = { + "type": "object", + "required": ["pipelines", "pagination"], + "properties": { + "pipelines": { + "type": "array", + "items": { + "type": "object", + "required": ["id"], + "properties": {"id": {"type": "string"}}, + }, + }, + "pagination": {"type": "object"}, + }, +} + +# GET /v2/{org}/pipelines/{id}/status — the router reads `status`. +PIPELINE_STATUS = { + "type": "object", + "required": ["status"], + "properties": {"status": {"type": "string"}}, +} + +# POST /v2/{org}/outputs — the created output; the router reads `id`. +OUTPUT = { + "type": "object", + "required": ["id"], + "properties": {"id": {"type": "string"}}, +} diff --git a/conformance/servers/ts_server.mjs b/conformance/servers/ts_server.mjs index 2a25635..1ff2ae0 100644 --- a/conformance/servers/ts_server.mjs +++ b/conformance/servers/ts_server.mjs @@ -1,19 +1,31 @@ -// Boot the TypeScript router for conformance testing, pointed at the mock -// Monad. Imports the built standalone package (run `pnpm -C routers/typescript -// build` first). +// Boot the TypeScript router for conformance testing. In the default (mock) +// mode the harness points it at the in-memory mock via MONAD_API_BASE; in live +// mode the same knobs are set to real Monad staging credentials. Every value +// falls back to the mock fixture default, so `node ts_server.mjs` with only +// MONAD_API_BASE set behaves exactly as before. +// +// Build the standalone package first: `pnpm -C routers/typescript build`. import http from 'node:http'; import { createEmbedRouter } from '../../routers/typescript/dist/index.js'; +const env = (name, fallback) => process.env[name] ?? fallback; +// A provisioned id may be intentionally empty (live tenant with no store) → treat +// "" as "not provisioned" so the router falls back to a dev/null sink. +const store = env('MONAD_STORE_ID', 'out_store') || undefined; +const source = env('MONAD_SOURCE_ID', 'in_source') || undefined; +// Empty allow-list → expose the whole catalog (undefined), never the empty set. +const allow = env('MONAD_CATALOG_ALLOW', 'aws-cloudtrail,okta-systemlog') + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + const router = createEmbedRouter({ - apiKey: 'conf-key', + apiKey: env('MONAD_API_KEY', 'conf-key'), apiBase: process.env.MONAD_API_BASE, - frameOrigin: 'https://app.monad.com/embed', - getCustomerOrgID: () => 'org_conf', - getProvisionedComponents: () => ({ - destinationOutputId: 'out_store', - sourceInputId: 'in_source' - }), - catalogAllow: ['aws-cloudtrail', 'okta-systemlog'] + frameOrigin: env('MONAD_FRAME_ORIGIN', 'https://app.monad.com/embed'), + getCustomerOrgID: () => env('MONAD_ORG_ID', 'org_conf'), + getProvisionedComponents: () => ({ destinationOutputId: store, sourceInputId: source }), + catalogAllow: allow.length ? allow : undefined }); const port = Number(process.env.PORT || 8791); diff --git a/conformance/test_conformance.py b/conformance/test_conformance.py index 67712e2..7eda466 100644 --- a/conformance/test_conformance.py +++ b/conformance/test_conformance.py @@ -8,6 +8,7 @@ import os +import pytest import schemathesis from schemathesis.specs.openapi.checks import ( content_type_conformance, @@ -20,6 +21,11 @@ ) _BASE_URL = os.environ.get("ROUTER_BASE_URL", "http://127.0.0.1:8791") +# Against real Monad (live mode) the fuzzer must not create resources with +# generated payloads — restrict it to read-only GET operations, which are also +# exactly the response-shape surface that has bitten us (catalog/connectors). +LIVE = os.environ.get("MONAD_LIVE") == "1" + schema = schemathesis.openapi.from_path(_SPEC) # We validate RESPONSE conformance: every response the router returns for every @@ -39,4 +45,6 @@ @schema.parametrize() def test_response_conforms_to_spec(case): + if LIVE and case.method.upper() != "GET": + pytest.skip("live mode fuzzes read-only GET operations only (no real mutations)") case.call_and_validate(base_url=_BASE_URL, checks=_CONFORMANCE_CHECKS) diff --git a/conformance/test_mock_fidelity.py b/conformance/test_mock_fidelity.py new file mode 100644 index 0000000..7fd4aeb --- /dev/null +++ b/conformance/test_mock_fidelity.py @@ -0,0 +1,102 @@ +"""Pin the stateful mock to the documented Monad response shapes. + +Boots its own mock instance and drives it exactly as the routers do, asserting +every response validates against the vendored Monad schemas (``monad_schemas``). +This is the hermetic complement to live conformance: live testing catches where +the *docs* are wrong (e.g. null-when-empty), while this catches the *mock* +drifting from the docs on a field a router parses — the class of gap that let +the missing ``pagination`` sibling ship. Fast, offline, no credentials. +""" + +import httpx +import pytest +from jsonschema import Draft202012Validator + +import mock_monad +import monad_schemas as S + +_ORG = "org_conf" + + +@pytest.fixture(scope="module") +def base_url(): + # A dedicated mock instance on its own port — independent of the conftest + # `servers` fixture (which also boots a router), so this stays hermetic in + # both mock and live router modes. + port = 8795 + server, _state = mock_monad.start(port) + try: + yield f"http://127.0.0.1:{port}" + finally: + server.shutdown() + + +@pytest.fixture() +def http(base_url): + with httpx.Client(base_url=base_url, timeout=10) as c: + yield c + + +def _valid(schema: dict, instance) -> None: + errors = sorted(Draft202012Validator(schema).iter_errors(instance), key=lambda e: e.path) + assert not errors, "; ".join(f"{list(e.path)}: {e.message}" for e in errors) + + +def test_session_shape(http): + r = http.post("/v3/sessions", json={"organization_id": _ORG, "ttl_seconds": 1800}) + assert r.status_code == 200 + _valid(S.SESSION, r.json()) + + +@pytest.mark.parametrize("kind", ["input", "output"]) +def test_catalog_shape(http, kind): + r = http.get(f"/v1/{kind}s") + assert r.status_code == 200 + _valid(S.CATALOG, r.json()) + + +@pytest.mark.parametrize("kind", ["input", "output"]) +def test_connectors_list_shape(http, kind): + # Covers both the populated (inputs) and the null-when-empty (outputs) case, + # each of which must still carry the `pagination` sibling. + r = http.get(f"/v1/{_ORG}/{kind}s?limit=1000&offset=0") + assert r.status_code == 200 + _valid(S.connectors_list(kind), r.json()) + + +def test_output_create_shape(http): + r = http.post(f"/v2/{_ORG}/outputs", json={"output_type": "dev-null", "name": "sink"}) + assert r.status_code == 200 + _valid(S.OUTPUT, r.json()) + + +def test_pipeline_lifecycle_shapes(http): + body = { + "name": "fidelity", + "enabled": True, + "nodes": [ + {"slug": "in", "component_id": "in_fidelity", "component_type": "input", "enabled": True}, + {"slug": "out", "component_id": "out_fidelity", "component_type": "output", "enabled": True}, + ], + "edges": [], + } + created = http.post(f"/v2/{_ORG}/pipelines/", json=body) + assert created.status_code == 201, created.text + _valid(S.PIPELINE, created.json()) + pid = created.json()["id"] + + listed = http.get(f"/v2/{_ORG}/pipelines/") + assert listed.status_code == 200 + _valid(S.PIPELINE_LIST, listed.json()) + + detail = http.get(f"/v2/{_ORG}/pipelines/{pid}") + assert detail.status_code == 200 + _valid(S.PIPELINE, detail.json()) + + status = http.get(f"/v2/{_ORG}/pipelines/{pid}/status") + assert status.status_code == 200 + _valid(S.PIPELINE_STATUS, status.json()) + + patched = http.patch(f"/v2/{_ORG}/pipelines/{pid}", json={**body, "enabled": False}) + assert patched.status_code == 200 + _valid(S.PIPELINE, patched.json()) diff --git a/conformance/test_scenarios.py b/conformance/test_scenarios.py index 6a8fef0..6e135b1 100644 --- a/conformance/test_scenarios.py +++ b/conformance/test_scenarios.py @@ -3,6 +3,18 @@ Runs against whichever router the ``servers`` fixture booted, over HTTP, so it is identical for every language. + +The fixture values are read from the **same env vars the conformance servers +read**, so the identical scenarios run two ways: + +- **mock mode** (default) — the env vars are unset, so everything falls back to + the in-memory mock's fixtures (``org_conf`` / ``out_store`` / ``in_conf`` …). +- **live mode** (``MONAD_LIVE=1``) — the env points the router at real Monad + staging, and the expected values come from the real tenant. The read-path + scenarios (config/session/catalog/connectors) run as-is — that is exactly the + response-shape surface that has bitten us. The mutating lifecycle scenarios + additionally require ``MONAD_LIVE_MUTATE=1`` + real ``CONF_INPUT_ID`` / + ``CONF_OUTPUT_ID`` throwaway components, and clean up after themselves. """ import os @@ -12,10 +24,32 @@ _BASE_URL = os.environ.get("ROUTER_BASE_URL", "http://127.0.0.1:8791") +LIVE = os.environ.get("MONAD_LIVE") == "1" +# Mutating scenarios against real Monad are opt-in (they create + delete real +# pipelines) and need throwaway components to wire. +MUTATE = os.environ.get("MONAD_LIVE_MUTATE") == "1" + +# Expected values — mirror the conformance servers' env defaults. +ORG = os.environ.get("MONAD_ORG_ID", "org_conf") +STORE = os.environ.get("MONAD_STORE_ID", "out_store") # ingress target; "" → dev/null sink +FRAME = os.environ.get("MONAD_FRAME_ORIGIN", "https://app.monad.com/embed") +# A real configured input/output to wire in the mutating lifecycle scenarios. +INPUT_ID = os.environ.get("CONF_INPUT_ID", "in_conf") +OUTPUT_ID = os.environ.get("CONF_OUTPUT_ID", "out_conf") +# The allow-list the catalog should be constrained to (empty → whole catalog). +_ALLOW = os.environ.get("MONAD_CATALOG_ALLOW", "aws-cloudtrail,okta-systemlog") +EXPECT_CATALOG = {s.strip() for s in _ALLOW.split(",") if s.strip()} + +# Skip a mutating scenario in live mode unless it was explicitly opted into. +skip_mutation = pytest.mark.skipif( + LIVE and not MUTATE, + reason="mutating live scenario — set MONAD_LIVE_MUTATE=1 with throwaway CONF_INPUT_ID/CONF_OUTPUT_ID", +) + @pytest.fixture() def client(): - with httpx.Client(base_url=_BASE_URL, timeout=15) as c: + with httpx.Client(base_url=_BASE_URL, timeout=30) as c: yield c @@ -23,65 +57,99 @@ def test_config_is_public(client): r = client.get("/embed/config") assert r.status_code == 200 body = r.json() - assert body["frameOrigin"] == "https://app.monad.com/embed" + assert body["frameOrigin"] == FRAME assert "apiBase" in body +def test_session_mints_for_tenant(client): + r = client.post("/embed/session") + assert r.status_code == 200, r.text + body = r.json() + assert body["organizationId"] == ORG + assert body.get("sessionToken") # a real (or mock) token was minted + + +def test_connectors_list_parses(client): + # The response-shape surface that has bitten us live (null-when-empty + + # a `pagination` sibling). It must parse into the contract array for both + # kinds regardless of how many the tenant has. + for kind in ("input", "output"): + r = client.get("/embed/connectors", params={"kind": kind}) + assert r.status_code == 200, r.text + rows = r.json() + assert isinstance(rows, list) + for row in rows: + assert {"id", "typeId", "name"} <= row.keys() + + +def test_catalog_is_allow_listed(client): + r = client.get("/embed/catalog", params={"kind": "input"}) + assert r.status_code == 200 + type_ids = {t["typeId"] for t in r.json()} + assert type_ids, "catalog must not be empty" + if EXPECT_CATALOG: + # The allow-list must constrain the catalog to (a subset of) itself. + assert type_ids <= EXPECT_CATALOG + + +@skip_mutation def test_ingress_lifecycle(client): # 1) mint a session for the resolved tenant r = client.post("/embed/session") assert r.status_code == 200, r.text - assert r.json()["organizationId"] == "org_conf" + assert r.json()["organizationId"] == ORG # 2) the iframe returned an input id → build the ingress pipeline - r = client.post("/embed/pipelines/ingress", json={"inputId": "in_conf", "name": "CloudTrail"}) + r = client.post("/embed/pipelines/ingress", json={"inputId": INPUT_ID, "name": "CloudTrail"}) assert r.status_code == 201, r.text built = r.json() pipeline_id = built["pipelineId"] - assert built["outputId"] == "out_store" # wired to the provisioned store + if STORE: + assert built["outputId"] == STORE # wired to the provisioned store + else: + assert built["outputId"] # a dev/null sink was created assert built["active"] is True # 3) status resolves the pipeline from the input id - r = client.get("/embed/pipelines", params={"connectorId": "in_conf", "kind": "input"}) + r = client.get("/embed/pipelines", params={"connectorId": INPUT_ID, "kind": "input"}) assert r.status_code == 200, r.text status = r.json() assert status["hasPipeline"] is True assert status["pipelineId"] == pipeline_id - assert status["outputId"] == "out_store" + if STORE: + assert status["outputId"] == STORE assert status["enabled"] is True # 4) disable — stops flow without deleting config r = client.post("/embed/pipelines/state", json={"pipelineId": pipeline_id, "enabled": False}) assert r.status_code == 204, r.text - r = client.get("/embed/pipelines", params={"connectorId": "in_conf", "kind": "input"}) + r = client.get("/embed/pipelines", params={"connectorId": INPUT_ID, "kind": "input"}) assert r.json()["enabled"] is False # 5) remove — the provisioned store is kept, the pipeline + input are gone - r = client.post("/embed/pipelines/remove", json={"connectorId": "in_conf", "kind": "input"}) + r = client.post("/embed/pipelines/remove", json={"connectorId": INPUT_ID, "kind": "input"}) assert r.status_code == 204, r.text - r = client.get("/embed/pipelines", params={"connectorId": "in_conf", "kind": "input"}) + r = client.get("/embed/pipelines", params={"connectorId": INPUT_ID, "kind": "input"}) assert r.json()["hasPipeline"] is False +@skip_mutation def test_egress_builds_from_provisioned_source(client): # the iframe returned an output id → wire the tenant's source → it - r = client.post("/embed/pipelines/egress", json={"outputId": "out_conf", "name": "Splunk"}) + r = client.post("/embed/pipelines/egress", json={"outputId": OUTPUT_ID, "name": "Splunk"}) assert r.status_code == 201, r.text - assert r.json()["outputId"] == "out_conf" - - -def test_catalog_is_allow_listed(client): - r = client.get("/embed/catalog", params={"kind": "input"}) - assert r.status_code == 200 - type_ids = {t["typeId"] for t in r.json()} - assert type_ids == {"aws-cloudtrail", "okta-systemlog"} + assert r.json()["outputId"] == OUTPUT_ID + if LIVE: + # clean up the pipeline (+ output) we just created; keep the shared source. + client.post("/embed/pipelines/remove", json={"connectorId": OUTPUT_ID, "kind": "output"}) # Every route that takes `kind` must reject an invalid value with the shared # error model — not just /catalog. (Schemathesis only ever sends enum-valid -# `kind`s, so this negative case has to be asserted explicitly.) +# `kind`s, so this negative case has to be asserted explicitly.) Router-local +# validation, so it runs identically against mock and live. @pytest.mark.parametrize( "call", [ @@ -100,15 +168,21 @@ def test_invalid_kind_is_rejected(client, call): def test_state_on_unknown_pipeline_is_404(client): # Toggling a pipeline that doesn't exist for this tenant is a 404 not_found — - # the router must translate Monad's 404, not fold it into a generic 502. + # the router must translate Monad's 404, not fold it into a generic 502. The + # router reads the pipeline before writing, so this creates nothing → safe live. r = client.post( "/embed/pipelines/state", - json={"pipelineId": "pipe_does_not_exist", "enabled": False}, + json={"pipelineId": "00000000-0000-4000-8000-000000000000", "enabled": False}, ) assert r.status_code == 404, r.text assert r.json()["code"] == "not_found" +@pytest.mark.skipif( + LIVE, + reason="Monad does not document a 409 on duplicate pipeline creation (201/400/500 only); " + "the mock emulates the constraint, real behavior is asserted only hermetically", +) def test_duplicate_ingress_conflicts(client): # Connecting the same source twice collides with existing state → 409 conflict # (a known Monad constraint the router must surface as the contract's `conflict`). From 62ae4ec9426fe1205364df08366ff94caa3cb95e Mon Sep 17 00:00:00 2001 From: curtis Date: Tue, 4 Aug 2026 11:56:20 -0400 Subject: [PATCH 4/7] fix(conformance): make the mock reproduce Monad's paging + component_of [PRO-455] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mock returned every row regardless of `limit` and had no `GET /v1/{org}/{kind}s/{id}` route at all, so it could not catch two bugs the routers actually have: they never send `limit` (Monad defaults it to 10) and they walk the whole pipeline list to answer a question `component_of` answers in one call. - page every list at Monad's default limit of 10, with no maximum - serve the connector detail route, including `component_of` — pinned to the same narrow projection the datastore emits (no `status`, no `nodes`) - accept `type` on output create, `output_type` staying a deprecated alias - track connectors statefully so a delete is observable - add the >10-pipelines scenario; it fails against the pre-fix routers --- conformance/mock_monad.py | 196 +++++++++++++++++++++++++----- conformance/monad_schemas.py | 36 ++++++ conformance/test_mock_fidelity.py | 52 +++++++- conformance/test_scenarios.py | 58 ++++++++- 4 files changed, 305 insertions(+), 37 deletions(-) diff --git a/conformance/mock_monad.py b/conformance/mock_monad.py index 2fec298..e89e5ef 100644 --- a/conformance/mock_monad.py +++ b/conformance/mock_monad.py @@ -11,12 +11,20 @@ - ``GET /v1/{kind}s`` (catalog) → a bare array of connector types. - ``GET /v1/{org}/{kind}s`` → ``{ "s": [...] | null, "pagination": {...} }`` (the list is **null**, not ``[]``, when empty). +- ``GET /v1/{org}/{kind}s/{id}`` → the connector record + ``component_of``, + the pipelines it is a node of. - ``POST /v2/{org}/pipelines`` → **201** with the full pipeline object. - ``GET /v2/{org}/pipelines`` → ``{ "pipelines": [...], "pagination": {...} }``. - ``GET /v2/{org}/pipelines/{id}`` → the full pipeline object at top level (``nodes``/``edges``/``enabled`` — no ``config`` wrapper). - ``POST /v2/{org}/outputs`` → the created output object. - ``POST /v3/sessions`` → ``{ "session_token", "expires_at" }``. + +**Pagination is enforced, and the default limit is 10** — matching every list +handler in Monad (``api/pkg/routes/v2/routes/pipelines.go:727`` and friends). +A router that omits ``limit`` gets 10 rows here exactly as it would in +production, which is what turns the "only ever sees 10" class of bug into a +test failure instead of a live incident. """ from __future__ import annotations @@ -24,13 +32,71 @@ import json import re import threading +import urllib.parse from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +# Monad's list handlers all default to 10 and enforce no maximum. +DEFAULT_LIMIT = 10 + +# Enough configured inputs that a router which does not page sees a truncated +# list. `test_connectors_pagination_is_exhaustive` pins the full count. +_SEEDED_INPUTS = 12 + +# Inputs the bulk-pipeline scenario wires, one pipeline each, to push the +# pipeline list past its first page. +_BULK_INPUTS = 12 + + +def _seed_connectors() -> dict[str, dict[str, dict]]: + """The tenant's configured connectors, keyed by id. + + `outputs` is deliberately empty so `GET /v1/{org}/outputs` returns **null** + — the real-Monad behavior its own spec gets wrong, and the reason live mode + exists. `out_store`/`out_conf` are the host's pre-provisioned components: + resolvable by id, but not part of the tenant's own list. + """ + inputs = { + f"cfg_{i}": { + "id": f"cfg_{i}", + "type": "aws-cloudtrail" if i % 2 else "okta-systemlog", + "name": f"Configured {i}", + "organization_id": "org_conf", + "config": {"settings": {}, "secrets": {}}, + } + for i in range(1, _SEEDED_INPUTS + 1) + } + # Ids the lifecycle scenarios wire, resolvable but outside the listed page + # so they cannot perturb the pagination counts. `bulk_*` exists to push the + # pipeline list past its first page. + extras = ["in_conf", "in_dup_conflict"] + [f"bulk_{i}" for i in range(1, _BULK_INPUTS + 1)] + for extra in extras: + inputs[extra] = { + "id": extra, + "type": "aws-cloudtrail", + "name": extra, + "organization_id": "org_conf", + "config": {"settings": {}, "secrets": {}}, + "listed": False, + } + outputs = { + oid: { + "id": oid, + "type": "dev-null", + "name": oid, + "organization_id": "org_conf", + "config": {"settings": {}, "secrets": {}}, + "listed": False, + } + for oid in ("out_store", "out_conf") + } + return {"input": inputs, "output": outputs} + class _State: def __init__(self) -> None: self.lock = threading.Lock() self.pipelines: dict[str, dict] = {} + self.connectors = _seed_connectors() self._counter = 0 def new_id(self, prefix: str) -> str: @@ -39,6 +105,16 @@ def new_id(self, prefix: str) -> str: return f"{prefix}_{self._counter}" +def _paginate(rows: list, limit: int, offset: int) -> tuple[list | None, dict]: + """Slice `rows` the way Monad does, and build the `pagination` sibling. + + An empty page comes back as ``None``, not ``[]`` — see the module note. + `total` is the full count, not the page length. + """ + page = rows[offset : offset + limit] + return (page or None), {"limit": limit, "offset": offset, "total": len(rows)} + + def _component_ids(pipeline: dict) -> frozenset: """The set of component ids a pipeline wires — used to detect a duplicate connection (Monad rejects connecting the same components twice → 409).""" @@ -84,11 +160,14 @@ def _pipeline_summary(p: dict) -> dict: def _output_view(oid: str, body: dict) -> dict: + # `type` is the canonical field; `output_type` is the deprecated alias the + # API still accepts, with `type` winning when both are sent + # (api/pkg/routes/v2/routes/organization_outputs.go:188). return { "id": oid, "name": body.get("name", ""), "description": body.get("description", ""), - "type": body.get("output_type", "dev-null"), + "type": body.get("type") or body.get("output_type") or "dev-null", "organization_id": "org_conf", "managed_by": "", "config": {"settings": {}, "secrets": {}}, @@ -97,6 +176,36 @@ def _output_view(oid: str, body: dict) -> dict: } +def _connector_view(record: dict) -> dict: + """A configured connector as the list/detail endpoints return it.""" + return {k: v for k, v in record.items() if k != "listed"} + + +def _component_of(pipelines: dict[str, dict], component_id: str) -> list[dict]: + """The pipelines a component is a node of — Monad's `component_of`. + + Deliberately the same narrow projection the datastore emits + (core/pkg/datastore/postgres/pipelines.go:292): **no `status`, no `nodes`**. + A router that needs the peer component has to fetch the pipeline detail. + """ + return [ + { + "id": p.get("id"), + "organization_id": "org_conf", + "name": p.get("name", ""), + "description": p.get("description", ""), + "enabled": bool(p.get("enabled")), + "component_tier": 0, + "input_id": "", + "managed_by": "", + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:00Z", + } + for p in pipelines.values() + if component_id in _component_ids(p) + ] + + def _make_handler(state: _State): class Handler(BaseHTTPRequestHandler): def log_message(self, *_args) -> None: # silence per-request logging @@ -133,6 +242,20 @@ def do_PATCH(self): # noqa: N802 def do_DELETE(self): # noqa: N802 self._route("DELETE") + def _limit_offset(self) -> tuple[int, int]: + """Parse `limit`/`offset` exactly as Monad's handlers do: anything + unparseable or out of range falls back to the default.""" + query = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query) + + def _int(name: str, default: int, floor: int) -> int: + try: + value = int(query.get(name, [""])[0]) + except (TypeError, ValueError): + return default + return value if value >= floor else default + + return _int("limit", DEFAULT_LIMIT, 1), _int("offset", 0, 0) + def _route(self, method: str) -> None: path = self.path.split("?", 1)[0] body = self._read() if method in ("POST", "PATCH") else {} @@ -166,35 +289,49 @@ def _route(self, method: str) -> None: ) # A tenant's configured connectors — wrapped alongside `pagination`, - # and returned as null (not []) when the tenant has none. Exercise - # both shapes: a populated `inputs`, an empty (null) `outputs`. + # paged with Monad's default limit of 10, and returned as null (not + # []) when the page is empty. Exercise both shapes: a populated + # `inputs` that spans two pages, an empty (null) `outputs`. m = re.fullmatch(r"/v1/([^/]+)/(inputs|outputs)", path) if method == "GET" and m: kind = m.group(2) - rows = ( - None - if kind == "outputs" - else [ - { - "id": "cfg_1", - "type": "aws-cloudtrail", - "name": "Configured", - "organization_id": "org_conf", - "config": {"settings": {}, "secrets": {}}, - } + limit, offset = self._limit_offset() + with state.lock: + all_rows = [ + _connector_view(c) + for c in state.connectors[kind[:-1]].values() + if c.get("listed", True) ] - ) - return self._send( - 200, - { - kind: rows, - "pagination": {"limit": 1000, "offset": 0, "total": 0 if rows is None else 1}, - }, - ) + rows, pagination = _paginate(all_rows, limit, offset) + return self._send(200, {kind: rows, "pagination": pagination}) + + # A single configured connector + `component_of`: the pipelines it is + # a node of. This is what lets a router resolve a connector's + # pipeline in one call instead of walking every pipeline in the org. + m = re.fullmatch(r"/v1/([^/]+)/(inputs|outputs)/([^/]+)", path) + if m and method in ("GET", "DELETE"): + kind, cid = m.group(2)[:-1], m.group(3) + with state.lock: + record = state.connectors[kind].get(cid) + if record is None: + return self._send(404, {"error": f"{kind} not found"}) + if method == "DELETE": + del state.connectors[kind][cid] + return self._send(204) + view = { + **_connector_view(record), + "component_of": _component_of(state.pipelines, cid), + } + return self._send(200, view) # Create an output (e.g. the dev/null sink) — the created record. + # It joins the tenant's connectors so it is resolvable by id and + # deletable, like any other output. if method == "POST" and re.fullmatch(r"/v2/([^/]+)/outputs", path): - return self._send(200, _output_view(state.new_id("out"), body)) + view = _output_view(state.new_id("out"), body) + with state.lock: + state.connectors["output"][view["id"]] = {**view, "listed": False} + return self._send(200, view) # pipelines collection (create / list) — check before status/detail if re.fullmatch(r"/v2/([^/]+)/pipelines/?", path): @@ -214,15 +351,11 @@ def _route(self, method: str) -> None: # Real Monad returns 201 with the full pipeline record. return self._send(201, view) if method == "GET": + limit, offset = self._limit_offset() with state.lock: items = [_pipeline_summary(p) for p in state.pipelines.values()] - return self._send( - 200, - { - "pipelines": items, - "pagination": {"limit": 1000, "offset": 0, "total": len(items)}, - }, - ) + page, pagination = _paginate(items, limit, offset) + return self._send(200, {"pipelines": page or [], "pagination": pagination}) if method == "GET" and re.fullmatch(r"/v2/([^/]+)/pipelines/([^/]+)/status", path): pid = path.rsplit("/", 2)[-2] @@ -259,9 +392,6 @@ def _route(self, method: str) -> None: state.pipelines.pop(pid, None) return self._send(204) - if method == "DELETE" and re.fullmatch(r"/v1/([^/]+)/(inputs|outputs)/([^/]+)", path): - return self._send(204) - return self._send(404, {"error": f"mock unhandled {method} {path}"}) return Handler diff --git a/conformance/monad_schemas.py b/conformance/monad_schemas.py index 5494bf1..af4cba1 100644 --- a/conformance/monad_schemas.py +++ b/conformance/monad_schemas.py @@ -18,6 +18,11 @@ ``null`` (not ``[]``) for a tenant with none, though its spec types it ``array``. Live conformance is what proves this, but pinning it here stops the mock from regressing to a non-null-emitting shape. +- ``component_of`` entries are pinned *without* ``status`` or ``nodes``: the + datastore projection behind them fills only a subset of ``models.Pipeline`` + (core/pkg/datastore/postgres/pipelines.go:292), so a router that tried to read + a pipeline's wiring straight off ``component_of`` would work against a + too-generous mock and fail live. """ # POST /v3/sessions — the embed session mint (swagger leaves the body untyped; @@ -69,6 +74,37 @@ def connectors_list(kind: str) -> dict: } +def connector_detail(kind: str) -> dict: + """GET /v1/{org}/{kind}s/{id} — the connector record plus ``component_of``, + the pipelines it is a node of. This is the one call that answers "which + pipeline is this connector wired into", so ``component_of`` is required + even when empty.""" + return { + "type": "object", + "required": ["id", "type", "name", "component_of"], + "properties": { + "id": {"type": "string"}, + "type": {"type": "string"}, + "name": {"type": "string"}, + "component_of": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "enabled"], + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "enabled": {"type": "boolean"}, + # Pinned absent — see the module note. + "status": False, + "nodes": False, + }, + }, + }, + }, + } + + # A pipeline node as the routers read it (match on component_type/component_id). _NODE = { "type": "object", diff --git a/conformance/test_mock_fidelity.py b/conformance/test_mock_fidelity.py index 7fd4aeb..211d229 100644 --- a/conformance/test_mock_fidelity.py +++ b/conformance/test_mock_fidelity.py @@ -64,10 +64,58 @@ def test_connectors_list_shape(http, kind): _valid(S.connectors_list(kind), r.json()) -def test_output_create_shape(http): - r = http.post(f"/v2/{_ORG}/outputs", json={"output_type": "dev-null", "name": "sink"}) +def test_connectors_list_defaults_to_ten(http): + # Monad's list handlers default `limit` to 10 with no maximum. A router that + # omits it gets a truncated page — the mock must reproduce that, not paper + # over it, or the "only ever sees 10" bug stays invisible until production. + unpaged = http.get(f"/v1/{_ORG}/inputs").json() + assert len(unpaged["inputs"]) == 10 + assert unpaged["pagination"] == {"limit": 10, "offset": 0, "total": mock_monad._SEEDED_INPUTS} + + rest = http.get(f"/v1/{_ORG}/inputs?limit=10&offset=10").json() + assert len(rest["inputs"]) == mock_monad._SEEDED_INPUTS - 10 + # Past the end the list is null, not [] — the same shape as "tenant has none". + assert http.get(f"/v1/{_ORG}/inputs?limit=10&offset=99").json()["inputs"] is None + + +@pytest.mark.parametrize("kind", ["input", "output"]) +def test_connector_detail_shape(http, kind): + cid = "in_conf" if kind == "input" else "out_store" + r = http.get(f"/v1/{_ORG}/{kind}s/{cid}") + assert r.status_code == 200, r.text + _valid(S.connector_detail(kind), r.json()) + # Unconnected component → an empty `component_of`, not a missing key. + assert r.json()["component_of"] == [] + assert http.get(f"/v1/{_ORG}/{kind}s/nope_404").status_code == 404 + + +def test_connector_detail_reports_component_of(http): + body = { + "name": "component-of", + "enabled": True, + "nodes": [ + {"slug": "in", "component_id": "cfg_3", "component_type": "input", "enabled": True}, + {"slug": "out", "component_id": "out_store", "component_type": "output", "enabled": True}, + ], + "edges": [], + } + pid = http.post(f"/v2/{_ORG}/pipelines/", json=body).json()["id"] + for cid in ("cfg_3", "out_store"): + kind = "inputs" if cid.startswith("cfg") else "outputs" + entry = http.get(f"/v1/{_ORG}/{kind}/{cid}").json()["component_of"] + assert [p["id"] for p in entry] == [pid] + assert entry[0]["enabled"] is True + http.delete(f"/v2/{_ORG}/pipelines/{pid}") + + +@pytest.mark.parametrize("field", ["type", "output_type"]) +def test_output_create_shape(http, field): + # `type` is canonical; `output_type` is the deprecated alias the API still + # accepts. Both must round-trip to the same created record. + r = http.post(f"/v2/{_ORG}/outputs", json={field: "dev-null", "name": "sink"}) assert r.status_code == 200 _valid(S.OUTPUT, r.json()) + assert r.json()["type"] == "dev-null" def test_pipeline_lifecycle_shapes(http): diff --git a/conformance/test_scenarios.py b/conformance/test_scenarios.py index 6e135b1..4c0a549 100644 --- a/conformance/test_scenarios.py +++ b/conformance/test_scenarios.py @@ -22,6 +22,8 @@ import httpx import pytest +import mock_monad + _BASE_URL = os.environ.get("ROUTER_BASE_URL", "http://127.0.0.1:8791") LIVE = os.environ.get("MONAD_LIVE") == "1" @@ -46,6 +48,13 @@ reason="mutating live scenario — set MONAD_LIVE_MUTATE=1 with throwaway CONF_INPUT_ID/CONF_OUTPUT_ID", ) +# Scenarios that assert against the mock's seeded fixtures (exact row counts, +# bulk pipeline creation). They pin router logic, not Monad behavior, so they +# run hermetically only — a real tenant has its own arbitrary inventory. +mock_only = pytest.mark.skipif( + LIVE, reason="asserts against the mock's seeded fixture counts; a live tenant has its own" +) + @pytest.fixture() def client(): @@ -82,6 +91,47 @@ def test_connectors_list_parses(client): assert {"id", "typeId", "name"} <= row.keys() +@mock_only +def test_connectors_pagination_is_exhaustive(client): + # Monad pages every list at limit=10 by default and the /embed contract + # returns a bare array, so the router owns draining the pages. A guard, not + # a gate: it fails a router that sends no `limit` at all, but it cannot + # distinguish exhaustive paging from a large hardcoded limit — the mock, + # like Monad, enforces no maximum. Correctness past a fixed ceiling is what + # live mode is for. + rows = client.get("/embed/connectors", params={"kind": "input"}).json() + assert len(rows) == mock_monad._SEEDED_INPUTS, ( + f"expected all {mock_monad._SEEDED_INPUTS} configured inputs, got {len(rows)} " + "— the router is not draining pages" + ) + assert len({r["id"] for r in rows}) == len(rows), "paging returned duplicates" + + +@mock_only +def test_status_resolves_past_the_first_page_of_pipelines(client): + # The regression gate for the "only ever sees 10 pipelines" bug: resolving a + # connector's pipeline must not depend on where that pipeline falls in the + # org's list. A router that scans `GET /v2/{org}/pipelines/` sees only the + # first page and reports hasPipeline:false for everything after it. + ids = [f"bulk_{i}" for i in range(1, 13)] + try: + for cid in ids: + r = client.post("/embed/pipelines/ingress", json={"inputId": cid, "name": cid}) + assert r.status_code == 201, r.text + + # The last one is well past page 1 (limit=10). + last = ids[-1] + r = client.get("/embed/pipelines", params={"connectorId": last, "kind": "input"}) + assert r.status_code == 200, r.text + assert r.json()["hasPipeline"] is True, ( + f"{last}'s pipeline was created but did not resolve — the router is " + "only seeing the first page of pipelines" + ) + finally: + for cid in ids: + client.post("/embed/pipelines/remove", json={"connectorId": cid, "kind": "input"}) + + def test_catalog_is_allow_listed(client): r = client.get("/embed/catalog", params={"kind": "input"}) assert r.status_code == 200 @@ -127,12 +177,16 @@ def test_ingress_lifecycle(client): r = client.get("/embed/pipelines", params={"connectorId": INPUT_ID, "kind": "input"}) assert r.json()["enabled"] is False - # 5) remove — the provisioned store is kept, the pipeline + input are gone + # 5) remove — the provisioned store is kept, the pipeline + input are gone. + # The input no longer exists upstream, so asking for its pipeline is a 404, + # not a 200 saying "no pipeline": the router resolves status by fetching the + # connector, and a deleted connector is genuinely not found. r = client.post("/embed/pipelines/remove", json={"connectorId": INPUT_ID, "kind": "input"}) assert r.status_code == 204, r.text r = client.get("/embed/pipelines", params={"connectorId": INPUT_ID, "kind": "input"}) - assert r.json()["hasPipeline"] is False + assert r.status_code == 404, r.text + assert r.json()["code"] == "not_found" @skip_mutation From 9dbe1af15ea5214925d3d8065a8638cbe14ed469 Mon Sep 17 00:00:00 2001 From: curtis Date: Tue, 4 Aug 2026 11:56:20 -0400 Subject: [PATCH 5/7] fix(embed): correct the TypeScript router against the real Monad API [PRO-455] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the Go port (#15) surfaced defects the TypeScript reference has too. Validated against the API source rather than the published spec, which is stale. - resolve a connector's pipeline via `GET /v1/{org}/{kind}s/{id}`'s `component_of` instead of scanning every pipeline. The scan sent no `limit`, so past the tenant's tenth pipeline it reported "not connected" for pipelines that exist. - `setEnabled` now PATCHes `{enabled}` alone. PATCH became a true partial update in api#2163; the read-modify-write it replaces silently dropped node `config_overrides` and edge `schema_detection_spec` on every toggle. - drain the connector list instead of one `limit=1000` request - send `type` on output create; `output_type` is deprecated (api#2156) - drop the speculative `data` / `config` response-shape fallbacks — Monad returns neither - map `kind` to its path segment explicitly rather than appending "s" --- routers/typescript/src/core.ts | 20 ++-- routers/typescript/src/monad.ts | 159 ++++++++++++++++---------------- 2 files changed, 84 insertions(+), 95 deletions(-) diff --git a/routers/typescript/src/core.ts b/routers/typescript/src/core.ts index 9170968..1bc7699 100644 --- a/routers/typescript/src/core.ts +++ b/routers/typescript/src/core.ts @@ -214,16 +214,7 @@ export function createEmbedHandler( throw new EmbedError(400, 'invalid_request', "Query 'connectorId' is required."); } const kind = requireKind(req.query.kind, "Query 'kind'"); - if (kind === 'input') { - return json(200, await upstream(monad.pipelineStatus(org, connectorId))); - } - const r = await upstream(monad.findPipelineByOutput(org, connectorId)); - return json(200, { - hasPipeline: r !== null, - enabled: r?.enabled ?? false, - pipelineId: r?.pipelineId, - inputId: r?.inputId - }); + return json(200, await upstream(monad.pipelineFor(org, kind, connectorId))); } if (method === 'POST' && path === '/pipelines/state') { @@ -241,8 +232,8 @@ export function createEmbedHandler( (req.body as Record | undefined)?.kind as string | undefined, "Field 'kind'" ); + const status = await upstream(monad.pipelineFor(org, kind, connectorId)); if (kind === 'input') { - const status = await upstream(monad.pipelineStatus(org, connectorId)); const prov = await provision(org); const keepStore = Boolean( prov.destinationOutputId && @@ -257,10 +248,13 @@ export function createEmbedHandler( ) ); } else { - const r = await upstream(monad.findPipelineByOutput(org, connectorId)); // Keep the shared source input; remove only the pipeline + output. await upstream( - monad.remove(org, { pipelineId: r?.pipelineId, outputId: connectorId }, { input: false }) + monad.remove( + org, + { pipelineId: status.pipelineId, outputId: connectorId }, + { input: false } + ) ); } return noContent(); diff --git a/routers/typescript/src/monad.ts b/routers/typescript/src/monad.ts index e1bb381..c84e7b3 100644 --- a/routers/typescript/src/monad.ts +++ b/routers/typescript/src/monad.ts @@ -51,10 +51,19 @@ export interface CleanupPolicy { const POLL_ATTEMPTS = 15; const POLL_INTERVAL_MS = 2000; +/** Rows per request when draining a Monad list. Monad defaults `limit` to 10 + * and enforces no maximum, so this is purely a round-trip/response-size + * trade-off — correctness comes from draining, not from the size. */ +const PAGE_SIZE = 200; + /** URL-encode a single path segment so browser-supplied ids can't inject query * params or traverse the path (`/`, `?`, `#` are neutralised). */ const seg = (s: string): string => encodeURIComponent(s); +/** The contract's singular `kind` → Monad's collection path segment. Explicit + * rather than `${kind}s`, so the mapping is a fact you can read. */ +const collection = (kind: ComponentKind): string => (kind === 'input' ? 'inputs' : 'outputs'); + /** * Thrown when a Monad API call fails. Carries the status + internal detail for * server-side logging; the router surfaces only the generic super() message to @@ -108,20 +117,37 @@ export class MonadApi { } async listCatalog(kind: ComponentKind, allow?: string[]): Promise { - const types = (await this.req(`/v1/${kind}s`)) as { type_id: string; name: string }[]; - const list = types.map((t) => ({ typeId: t.type_id, name: t.name })); + const types = (await this.req(`/v1/${collection(kind)}`)) as { + type_id: string; + name: string; + }[]; + const list = (types ?? []).map((t) => ({ typeId: t.type_id, name: t.name })); if (!allow || allow.length === 0) return list; const set = new Set(allow); return list.filter((t) => set.has(t.typeId)); } + /** + * Every configured connector of a kind. Monad pages this at `limit=10` by + * default with no maximum, and the `/embed` contract returns a bare array + * with no pagination — so the router owns draining the pages. Sending one + * large `limit` instead would silently truncate whichever tenant outgrows it. + */ async listConnectors(org: string, kind: ComponentKind): Promise { - const page = (await this.req(`/v1/${seg(org)}/${kind}s?limit=1000&offset=0`)) as Record< - string, - { id: string; type: string; name: string }[] - >; - const rows = page?.[`${kind}s`] ?? []; - return rows.map((r) => ({ id: r.id, typeId: r.type, name: r.name })); + const key = collection(kind); + const rows: ConfiguredConnector[] = []; + for (let offset = 0; ; offset += PAGE_SIZE) { + const page = await this.req(`/v1/${seg(org)}/${key}?limit=${PAGE_SIZE}&offset=${offset}`); + // Monad returns `null`, not `[]`, for a page with no rows. + const items = (page?.[key] ?? []) as { id: string; type: string; name: string }[]; + rows.push(...items.map((r) => ({ id: r.id, typeId: r.type, name: r.name }))); + // A short page is the last page; `total` just lets us stop one round + // trip earlier when the count divides evenly. + if (items.length < PAGE_SIZE) break; + const total = page?.pagination?.total; + if (typeof total === 'number' && rows.length >= total) break; + } + return rows; } private async wire( @@ -167,7 +193,8 @@ export class MonadApi { const output = (await this.req(`/v2/${seg(org)}/outputs`, { method: 'POST', body: JSON.stringify({ - output_type: 'dev-null', + // `type` is the canonical field; `output_type` is a deprecated alias. + type: 'dev-null', name: `${name} → /dev/null`, description: 'Auto-created sink for embed pipeline', promise_id: '', @@ -194,88 +221,56 @@ export class MonadApi { return this.wire(org, opts.fromInputId, opts.outputId, opts.name); } - private async pipelines(org: string): Promise { - const listed = await this.req(`/v2/${seg(org)}/pipelines/`); - return Array.isArray(listed) ? listed : (listed?.pipelines ?? listed?.data ?? []); - } - private async detail(org: string, id: string): Promise { - const d = await this.req(`/v2/${seg(org)}/pipelines/${seg(id)}`); - return d?.config ?? d ?? {}; + return (await this.req(`/v2/${seg(org)}/pipelines/${seg(id)}`)) ?? {}; } - async pipelineStatus(org: string, inputId: string): Promise { - for (const summary of await this.pipelines(org)) { - if (!summary?.id) continue; - let p: any; - try { - p = await this.detail(org, summary.id); - } catch { - continue; - } - const nodes: any[] = p.nodes ?? []; - const inNode = nodes.find((n) => n.component_type === 'input' && n.component_id === inputId); - if (!inNode) continue; - const outNode = nodes.find((n) => n.component_type === 'output'); - return { - hasPipeline: true, - enabled: Boolean(p.enabled), - pipelineId: summary.id, - outputId: outNode?.component_id - }; - } - return { hasPipeline: false, enabled: false }; - } + /** + * The pipeline a configured connector is wired into. + * + * Monad answers this directly: `GET /v1/{org}/{kind}s/{id}` returns + * `component_of`, the pipelines the component is a node of. Walking the + * org's pipeline list instead would be both O(n) and wrong — that list pages + * at 10, so any pipeline past the first page would read as "not connected". + * + * `component_of` carries no wiring (the datastore fills only a summary + * projection), so resolving the peer connector costs one further fetch. + * Throws `UpstreamError(404)` when the connector itself does not exist. + */ + async pipelineFor( + org: string, + kind: ComponentKind, + connectorId: string + ): Promise { + const connector = await this.req(`/v1/${seg(org)}/${collection(kind)}/${seg(connectorId)}`); + const [pipeline] = (connector?.component_of ?? []) as any[]; + if (!pipeline?.id) return { hasPipeline: false, enabled: false }; - async findPipelineByOutput(org: string, outputId: string): Promise { - for (const summary of await this.pipelines(org)) { - if (!summary?.id) continue; - let p: any; - try { - p = await this.detail(org, summary.id); - } catch { - continue; - } - const nodes: any[] = p.nodes ?? []; - const outNode = nodes.find( - (n) => n.component_type === 'output' && n.component_id === outputId - ); - if (!outNode) continue; - const inNode = nodes.find((n) => n.component_type === 'input'); - return { - hasPipeline: true, - enabled: Boolean(p.enabled), - pipelineId: summary.id, - inputId: inNode?.component_id - }; - } - return null; + const peerType = kind === 'input' ? 'output' : 'input'; + const nodes: any[] = (await this.detail(org, pipeline.id)).nodes ?? []; + const peer = nodes.find((n) => n.component_type === peerType)?.component_id; + + return { + hasPipeline: true, + enabled: Boolean(pipeline.enabled), + pipelineId: pipeline.id, + ...(kind === 'input' ? { outputId: peer } : { inputId: peer }) + }; } + /** + * Flip a pipeline's enabled flag. + * + * `PATCH` is a true partial update: omitted fields keep their stored value + * and the node/edge graph is preserved untouched. Reading the pipeline and + * sending it back would replace the graph with whatever subset of fields the + * round trip happened to reproduce — silently dropping node + * `config_overrides` and edge `schema_detection_spec`. + */ async setEnabled(org: string, pipelineId: string, enabled: boolean): Promise { - const p = await this.detail(org, pipelineId); await this.req(`/v2/${seg(org)}/pipelines/${seg(pipelineId)}`, { method: 'PATCH', - body: JSON.stringify({ - name: p.name, - description: p.description ?? '', - enabled, - nodes: (p.nodes ?? []).map((n: any) => ({ - id: n.id, - slug: n.slug, - component_id: n.component_id, - component_type: n.component_type, - enabled: n.enabled ?? true - })), - edges: (p.edges ?? []).map((e: any) => ({ - name: e.name, - description: e.description ?? '', - from_node_instance_id: e.from_node_instance_id, - to_node_instance_id: e.to_node_instance_id, - disabled: e.disabled ?? false, - conditions: e.conditions - })) - }) + body: JSON.stringify({ enabled }) }); } From 1dc9b955097c3e87e563f266156ef22d639ad00a Mon Sep 17 00:00:00 2001 From: clippy Date: Tue, 8 Sep 2026 13:44:13 -0700 Subject: [PATCH 6/7] fix(embed): harden TS router pagination + body limits; align conformance CI to Node 24 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial-review fixes (A/B/C), TS layer: - listConnectors: drain until a SHORT page (the only safe stop); drop the pagination.total early-break that truncated the list when Monad under-reports total; add MAX_PAGES cap so an upstream that never returns a short page bounds the loop instead of spinning / exhausting memory. - node adapter: cap buffered request body at 1 MiB (bare-Node OOM guard) → 400 invalid_request when exceeded. - ci.yml: conformance job now uses node-version-file (.tool-versions → Node 24.19.0) instead of the stale node-version: 22, matching the build/spec jobs and the package engines pin. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_011ASKA2VrufrNLGGrcFjY7n --- .github/workflows/ci.yml | 2 +- routers/typescript/src/monad.ts | 20 ++++++++++++----- routers/typescript/src/node.ts | 38 +++++++++++++++++++++++++++------ 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87ef8af..1518462 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 22 + node-version-file: '.tool-versions' cache: pnpm - run: pnpm install --frozen-lockfile diff --git a/routers/typescript/src/monad.ts b/routers/typescript/src/monad.ts index c84e7b3..cd2c5aa 100644 --- a/routers/typescript/src/monad.ts +++ b/routers/typescript/src/monad.ts @@ -56,6 +56,12 @@ const POLL_INTERVAL_MS = 2000; * trade-off — correctness comes from draining, not from the size. */ const PAGE_SIZE = 200; +/** Hard ceiling on pages drained from a single list. Guards against an upstream + * that never returns a short page (e.g. an offset-clamping proxy) so the loop + * is bounded instead of spinning forever / exhausting memory. 1000 × PAGE_SIZE + * = 200k rows — far beyond any real tenant. */ +const MAX_PAGES = 1000; + /** URL-encode a single path segment so browser-supplied ids can't inject query * params or traverse the path (`/`, `?`, `#` are neutralised). */ const seg = (s: string): string => encodeURIComponent(s); @@ -136,16 +142,20 @@ export class MonadApi { async listConnectors(org: string, kind: ComponentKind): Promise { const key = collection(kind); const rows: ConfiguredConnector[] = []; - for (let offset = 0; ; offset += PAGE_SIZE) { + // A short page is the ONLY safe stop condition. Trusting `pagination.total` + // to break early truncates the list whenever Monad under-reports it, so we + // drain until a page comes back short. MAX_PAGES bounds a pathological + // upstream that never does. + for (let pageIndex = 0; ; pageIndex++) { + if (pageIndex >= MAX_PAGES) { + throw new UpstreamError(0, `pagination exceeded ${MAX_PAGES} pages draining ${key}`); + } + const offset = pageIndex * PAGE_SIZE; const page = await this.req(`/v1/${seg(org)}/${key}?limit=${PAGE_SIZE}&offset=${offset}`); // Monad returns `null`, not `[]`, for a page with no rows. const items = (page?.[key] ?? []) as { id: string; type: string; name: string }[]; rows.push(...items.map((r) => ({ id: r.id, typeId: r.type, name: r.name }))); - // A short page is the last page; `total` just lets us stop one round - // trip earlier when the count divides evenly. if (items.length < PAGE_SIZE) break; - const total = page?.pagination?.total; - if (typeof total === 'number' && rows.length >= total) break; } return rows; } diff --git a/routers/typescript/src/node.ts b/routers/typescript/src/node.ts index f2a686a..7eb90e9 100644 --- a/routers/typescript/src/node.ts +++ b/routers/typescript/src/node.ts @@ -12,6 +12,14 @@ import type { IncomingMessage, ServerResponse } from 'node:http'; import { createEmbedHandler, type EmbedRequest, type EmbedServerConfig } from './core'; +/** Cap on a buffered request body. Embed requests carry only ids + names, so + * 1 MiB is generous; the cap stops a hostile/buggy client (behind host auth) + * from driving the bare-Node adapter to OOM. */ +const MAX_BODY_BYTES = 1 << 20; + +/** Thrown when a request body exceeds {@link MAX_BODY_BYTES}. */ +class PayloadTooLargeError extends Error {} + export interface EmbedRouterOptions { /** * Path prefix to strip from the incoming URL before matching. Defaults to @@ -38,8 +46,15 @@ async function readJsonBody(req: IncomingMessage): Promise { if (method === 'GET' || method === 'HEAD') return undefined; const chunks: Buffer[] = []; + let size = 0; for await (const chunk of req) { - chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : (chunk as Buffer)); + const buf = typeof chunk === 'string' ? Buffer.from(chunk) : (chunk as Buffer); + size += buf.length; + if (size > MAX_BODY_BYTES) { + req.destroy(); + throw new PayloadTooLargeError(); + } + chunks.push(buf); } if (chunks.length === 0) return undefined; const text = Buffer.concat(chunks).toString('utf8').trim(); @@ -83,14 +98,23 @@ export function createEmbedRouter(config: EmbedServerConfig, options: EmbedRoute } res.setHeader('content-type', 'application/json; charset=utf-8'); res.end(JSON.stringify(result.body)); - })().catch(() => { - if (!res.headersSent) { - res.statusCode = 500; - res.setHeader('content-type', 'application/json; charset=utf-8'); - res.end(JSON.stringify({ code: 'internal_error', message: 'Request handling failed.' })); - } else { + })().catch((err) => { + if (res.headersSent) { res.end(); + return; } + // A too-large body maps to the contract's 400 invalid_request (413 is + // not a declared status); anything else is a generic 500. + const tooLarge = err instanceof PayloadTooLargeError; + res.statusCode = tooLarge ? 400 : 500; + res.setHeader('content-type', 'application/json; charset=utf-8'); + res.end( + JSON.stringify( + tooLarge + ? { code: 'invalid_request', message: 'Request body exceeds the maximum size.' } + : { code: 'internal_error', message: 'Request handling failed.' } + ) + ); }); }; } From bbe54d77900b2369f3b81d25203d6bebe7bd9a98 Mon Sep 17 00:00:00 2001 From: clippy Date: Tue, 8 Sep 2026 14:21:29 -0700 Subject: [PATCH 7/7] =?UTF-8?q?fix(embed):=20review=20round=202=20?= =?UTF-8?q?=E2=80=94=20orphan-safety,=20redaction,=20docs,=20conformance?= =?UTF-8?q?=20gaps=20(TS=20+=20suite)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Router (TS): - wire: a transient status-poll failure no longer fails the request — the pipeline already exists, so report it as built (last-known status) instead of orphaning it and inviting a duplicate on retry. (D) - buildDevNull: best-effort delete the just-created sink if wiring fails, so a retry doesn't accumulate dev/null outputs. (E) - pipelineFor: document the one-pipeline-per-connector invariant behind component_of[0]. (F) - core resolve: redact the host getCustomerOrgID error (log server-side, return a generic 401) — it could carry internal detail. (H) Docs: add docs/embed-security-and-operations.md — dedicated-org-per-tenant trust model (G) + non-atomic lifecycle, delete ordering, and the recommendation to alert/monitor for orphans (E). Conformance: - mock now seeds 205 inputs (> PAGE_SIZE 200) so draining MUST cross a page boundary; the exhaustive test becomes a real drain gate. (6) - add test_devnull.py: boots a no-store router and exercises the dev/null ingress path the main harness never hits. (7) - drop the tautological "past-first-page pipeline" test (routers resolve via component_of, not the pipeline list). (8) Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_011ASKA2VrufrNLGGrcFjY7n --- conformance/mock_monad.py | 21 +++++----- conformance/test_devnull.py | 57 +++++++++++++++++++++++++++ conformance/test_mock_fidelity.py | 6 ++- conformance/test_scenarios.py | 36 +++-------------- docs/embed-security-and-operations.md | 50 +++++++++++++++++++++++ routers/typescript/src/core.ts | 10 ++--- routers/typescript/src/monad.ts | 35 ++++++++++++++-- 7 files changed, 162 insertions(+), 53 deletions(-) create mode 100644 conformance/test_devnull.py create mode 100644 docs/embed-security-and-operations.md diff --git a/conformance/mock_monad.py b/conformance/mock_monad.py index e89e5ef..468ba8f 100644 --- a/conformance/mock_monad.py +++ b/conformance/mock_monad.py @@ -38,13 +38,12 @@ # Monad's list handlers all default to 10 and enforce no maximum. DEFAULT_LIMIT = 10 -# Enough configured inputs that a router which does not page sees a truncated -# list. `test_connectors_pagination_is_exhaustive` pins the full count. -_SEEDED_INPUTS = 12 - -# Inputs the bulk-pipeline scenario wires, one pipeline each, to push the -# pipeline list past its first page. -_BULK_INPUTS = 12 +# More configured inputs than fit in one page (every router requests +# PAGE_SIZE=200), so draining MUST advance `offset` across a page boundary to +# see them all. A router that fails to page sees a truncated list and +# `test_connectors_pagination_is_exhaustive` fails — turning the "only ever sees +# the first page" class of bug into a test failure instead of a live incident. +_SEEDED_INPUTS = 205 def _seed_connectors() -> dict[str, dict[str, dict]]: @@ -65,10 +64,10 @@ def _seed_connectors() -> dict[str, dict[str, dict]]: } for i in range(1, _SEEDED_INPUTS + 1) } - # Ids the lifecycle scenarios wire, resolvable but outside the listed page - # so they cannot perturb the pagination counts. `bulk_*` exists to push the - # pipeline list past its first page. - extras = ["in_conf", "in_dup_conflict"] + [f"bulk_{i}" for i in range(1, _BULK_INPUTS + 1)] + # Ids the lifecycle scenarios wire, resolvable but marked unlisted so they + # don't perturb the pagination counts the drain test pins. `in_devnull` is + # the throwaway the no-store dev/null test wires. + extras = ["in_conf", "in_dup_conflict", "in_devnull"] for extra in extras: inputs[extra] = { "id": extra, diff --git a/conformance/test_devnull.py b/conformance/test_devnull.py new file mode 100644 index 0000000..acb8b6f --- /dev/null +++ b/conformance/test_devnull.py @@ -0,0 +1,57 @@ +"""Dev/null ingress coverage — the no-provisioned-store path. + +The main harness always provisions a destination store (MONAD_STORE_ID=out_store), +so a router's ``buildDevNull`` branch — create a /dev/null sink, then wire the +input to it — is never exercised there. This boots a SECOND router configured +with no store, pointed at the same mock, and drives one ingress through the +dev/null path. + +Mock-only: in live mode we don't fabricate a no-store tenant. +""" + +import os +import signal +import subprocess + +import httpx +import pytest + +# Reuse the harness's per-language boot machinery (cmd/cwd/env + health wait). +from conftest import _command, _wait_healthy + +LIVE = os.environ.get("MONAD_LIVE") == "1" +_PORT = 8796 # distinct from the main router's ROUTER_PORT (8791) + +pytestmark = pytest.mark.skipif(LIVE, reason="dev/null path is exercised in mock mode only") + + +@pytest.fixture(scope="module") +def devnull_base_url(): + cmd, cwd, env = _command() + env = dict(env) + env["MONAD_STORE_ID"] = "" # no provisioned store → the router builds a dev/null sink + env["PORT"] = str(_PORT) + proc = subprocess.Popen(cmd, cwd=cwd, env=env, start_new_session=True) + try: + if not _wait_healthy(f"http://127.0.0.1:{_PORT}/embed/config"): + raise RuntimeError("no-store router did not become healthy") + yield f"http://127.0.0.1:{_PORT}" + finally: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + except ProcessLookupError: + pass + + +def test_ingress_builds_devnull_sink(devnull_base_url): + with httpx.Client(base_url=devnull_base_url, timeout=30) as c: + r = c.post("/embed/pipelines/ingress", json={"inputId": "in_devnull", "name": "DevNull"}) + assert r.status_code == 201, r.text + built = r.json() + # A sink was auto-created (a fresh id, not one of the host's provisioned + # stores) and the input wired to it. + assert built["outputId"], "no output was created for the dev/null pipeline" + assert built["outputId"] not in ("out_store", "out_conf") + assert built["active"] is True + # Clean up the pipeline + the auto-created sink. + c.post("/embed/pipelines/remove", json={"connectorId": "in_devnull", "kind": "input"}) diff --git a/conformance/test_mock_fidelity.py b/conformance/test_mock_fidelity.py index 211d229..2ae94c7 100644 --- a/conformance/test_mock_fidelity.py +++ b/conformance/test_mock_fidelity.py @@ -72,10 +72,12 @@ def test_connectors_list_defaults_to_ten(http): assert len(unpaged["inputs"]) == 10 assert unpaged["pagination"] == {"limit": 10, "offset": 0, "total": mock_monad._SEEDED_INPUTS} + # A page honors the requested limit (not the count remaining). rest = http.get(f"/v1/{_ORG}/inputs?limit=10&offset=10").json() - assert len(rest["inputs"]) == mock_monad._SEEDED_INPUTS - 10 + assert len(rest["inputs"]) == 10 # Past the end the list is null, not [] — the same shape as "tenant has none". - assert http.get(f"/v1/{_ORG}/inputs?limit=10&offset=99").json()["inputs"] is None + beyond = mock_monad._SEEDED_INPUTS + 50 + assert http.get(f"/v1/{_ORG}/inputs?limit=10&offset={beyond}").json()["inputs"] is None @pytest.mark.parametrize("kind", ["input", "output"]) diff --git a/conformance/test_scenarios.py b/conformance/test_scenarios.py index 4c0a549..0dd851d 100644 --- a/conformance/test_scenarios.py +++ b/conformance/test_scenarios.py @@ -93,12 +93,11 @@ def test_connectors_list_parses(client): @mock_only def test_connectors_pagination_is_exhaustive(client): - # Monad pages every list at limit=10 by default and the /embed contract - # returns a bare array, so the router owns draining the pages. A guard, not - # a gate: it fails a router that sends no `limit` at all, but it cannot - # distinguish exhaustive paging from a large hardcoded limit — the mock, - # like Monad, enforces no maximum. Correctness past a fixed ceiling is what - # live mode is for. + # The mock seeds more configured inputs than fit in one page (every router + # requests PAGE_SIZE=200), so returning them all REQUIRES advancing `offset` + # across a page boundary. A router that fails to page — or stops early on an + # under-reported `total` — returns a truncated list and fails here. This is + # the "only ever sees the first page" regression gate. rows = client.get("/embed/connectors", params={"kind": "input"}).json() assert len(rows) == mock_monad._SEEDED_INPUTS, ( f"expected all {mock_monad._SEEDED_INPUTS} configured inputs, got {len(rows)} " @@ -107,31 +106,6 @@ def test_connectors_pagination_is_exhaustive(client): assert len({r["id"] for r in rows}) == len(rows), "paging returned duplicates" -@mock_only -def test_status_resolves_past_the_first_page_of_pipelines(client): - # The regression gate for the "only ever sees 10 pipelines" bug: resolving a - # connector's pipeline must not depend on where that pipeline falls in the - # org's list. A router that scans `GET /v2/{org}/pipelines/` sees only the - # first page and reports hasPipeline:false for everything after it. - ids = [f"bulk_{i}" for i in range(1, 13)] - try: - for cid in ids: - r = client.post("/embed/pipelines/ingress", json={"inputId": cid, "name": cid}) - assert r.status_code == 201, r.text - - # The last one is well past page 1 (limit=10). - last = ids[-1] - r = client.get("/embed/pipelines", params={"connectorId": last, "kind": "input"}) - assert r.status_code == 200, r.text - assert r.json()["hasPipeline"] is True, ( - f"{last}'s pipeline was created but did not resolve — the router is " - "only seeing the first page of pipelines" - ) - finally: - for cid in ids: - client.post("/embed/pipelines/remove", json={"connectorId": cid, "kind": "input"}) - - def test_catalog_is_allow_listed(client): r = client.get("/embed/catalog", params={"kind": "input"}) assert r.status_code == 200 diff --git a/docs/embed-security-and-operations.md b/docs/embed-security-and-operations.md new file mode 100644 index 0000000..8d768d8 --- /dev/null +++ b/docs/embed-security-and-operations.md @@ -0,0 +1,50 @@ +# Embed: security & operational model + +This applies to every backend router (`@monad-inc/embed-server`, and the Go and +Python routers) — they share one contract and one trust model. + +## Tenant isolation: one dedicated Monad org per tenant + +The host maps each incoming request to a Monad organization through the +`getCustomerOrgID` callback, and every Monad call the router makes is scoped to +that org. **Each embed tenant must get its own dedicated Monad org.** + +The router mounts _behind_ the host's authentication and treats the resolved org +as the whole authorization boundary: within that org, an embed user can list, +build, pause, and **remove** connectors and pipelines by id. There is no +per-object "was this created by embed?" check — that is deliberate, and it is +safe **only** because the org is dedicated to the tenant. + +> **Do not share an embed tenant's org with non-embed resources.** If a tenant's +> org also holds production inputs/outputs/pipelines created outside embed, an +> embed user could enumerate (`GET /embed/connectors`) and delete +> (`POST /embed/pipelines/remove`) them. Provision a separate org per embed +> tenant and keep host-managed infrastructure in a different org. + +Tenant-to-tenant isolation is never at risk: the org is resolved server-side, so +a browser cannot address another tenant's org. + +## Resource lifecycle is non-atomic — monitor for orphans + +Building or removing a connector is several Monad API calls in sequence +(create output → create pipeline → poll status; or delete pipeline → delete +connector → delete output). Monad exposes **no transaction and no idempotency +key**, so a failure part-way through can leave a resource behind: + +- **Create path** — if wiring the pipeline fails after a dev/null sink was + created, the router makes a **best-effort delete** of that sink. If the + compensating delete also fails, the sink is orphaned. +- **Delete path** — `remove` deletes in dependency order: **pipeline first** + (it references the others), then the connector, then the peer output. A + failure after the first delete leaves the remaining resources orphaned, and + because the pipeline that linked them is already gone, a retry cannot rediscover + them through the embed API. + +A transient failure while polling a freshly-created pipeline's status does **not** +orphan it: the pipeline already exists, so the router returns it as built (with +its last-known status) rather than failing and inviting a duplicate on retry. + +**Recommendation:** set up alerting/monitoring in each embed tenant's org for +orphaned resources — pipelines stuck non-running, and inputs/outputs that belong +to no pipeline — and reconcile them (delete or re-wire) out of band. This is the +backstop for the non-atomic delete path above. diff --git a/routers/typescript/src/core.ts b/routers/typescript/src/core.ts index 1bc7699..53bcff0 100644 --- a/routers/typescript/src/core.ts +++ b/routers/typescript/src/core.ts @@ -141,11 +141,11 @@ export function createEmbedHandler( return org; } catch (e) { if (e instanceof EmbedError) throw e; - throw new EmbedError( - 401, - 'unauthenticated', - e instanceof Error ? e.message : 'Tenant resolution failed.' - ); + // Redact the host callback's error detail — it can carry internal + // information (DB errors, internal URLs). Log it server-side; return a + // generic message, like the upstream-Monad path does. + console.error('[embed] getCustomerOrgID threw:', e); + throw new EmbedError(401, 'unauthenticated', 'Could not resolve a tenant for the request.'); } } diff --git a/routers/typescript/src/monad.ts b/routers/typescript/src/monad.ts index cd2c5aa..39d6ead 100644 --- a/routers/typescript/src/monad.ts +++ b/routers/typescript/src/monad.ts @@ -187,11 +187,20 @@ export class MonadApi { }) })) as { id: string }; + // The pipeline now EXISTS. Polling for `Running` is a courtesy — a + // transient status-poll failure must NOT fail the whole request, or the + // caller would treat a live pipeline as un-built and retry into a + // duplicate. On a poll error we stop and report the last-known status. let status = 'Pending'; for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { - const s = (await this.req(`/v2/${seg(org)}/pipelines/${seg(pipeline.id)}/status`)) as { - status?: string; - }; + let s: { status?: string }; + try { + s = (await this.req(`/v2/${seg(org)}/pipelines/${seg(pipeline.id)}/status`)) as { + status?: string; + }; + } catch { + break; + } status = s?.status ?? status; if (status === 'Running' || status === 'Erroring') break; await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); @@ -211,7 +220,21 @@ export class MonadApi { config: { settings: {}, secrets: {} } }) })) as { id: string }; - return this.wire(org, inputId, output.id, name); + try { + return await this.wire(org, inputId, output.id, name); + } catch (e) { + // The sink was created but wiring the pipeline failed. Best-effort + // delete the orphaned output so a retry doesn't accumulate dev/null + // sinks; if the cleanup itself fails, leave it for orphan monitoring + // (see docs/embed-security-and-operations.md) and surface the original + // error. Create/delete are non-atomic — Monad offers no transaction. + try { + await this.req(`/v1/${seg(org)}/outputs/${seg(output.id)}`, { method: 'DELETE' }); + } catch { + /* orphaned output — covered by the monitoring runbook */ + } + throw e; + } } /** Ingress: wire a configured input → the store, or a dev/null sink if none. */ @@ -253,6 +276,10 @@ export class MonadApi { connectorId: string ): Promise { const connector = await this.req(`/v1/${seg(org)}/${collection(kind)}/${seg(connectorId)}`); + // The embed flow wires each connector into exactly one pipeline (Monad + // guarantees a pipeline has a single input, and embed never adds a + // connector to a second pipeline), so `component_of` has at most one + // entry and taking the first is unambiguous. const [pipeline] = (connector?.component_of ?? []) as any[]; if (!pipeline?.id) return { hasPipeline: false, enabled: false };