diff --git a/README.md b/README.md index 1161748..593a466 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,32 @@ overrides (`allow_domains` / `deny_patterns` / `respect_robots`); `extract`, `bulk`, and `crawl` also accept a `retry` count for target timeouts, and `crawl` a `max_pages` budget. +### Structured output + +Every tool declares an `outputSchema` and returns `structuredContent` — typed +JSON, so an agent reads `status`, `completed`, `ok`, or `tokensSaved` as real +fields instead of pattern-matching them out of a rendered sentence: + +```json +{ + "kind": "crawl", + "jobId": "job_abc", + "status": "done", + "total": 12, + "completed": 12, + "done": true, + "truncated": false, + "results": [ + { "url": "https://example.com/a", "depth": 1, "ok": true, "markdown": "# A" }, + { "url": "https://example.com/b", "depth": 1, "ok": false, "error": "target_timeout" } + ] +} +``` + +The human-readable rendering is still returned in the usual text block, so +hosts that don't yet read structured output behave exactly as before. Errors +stay text-only and carry the API's stable error `code`. + > **Not exposed as tools:** API-key rotation (`/keys/rotate`) and webhook-secret > rotation (`/webhook/rotate`). Both are destructive and irreversible — the old > credential dies the instant the call returns, with no recovery flow — which is diff --git a/package.json b/package.json index 18e5bd7..212df60 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "start:http": "node dist/http.js", "dev": "tsc -p tsconfig.json --watch", "typecheck": "tsc -p tsconfig.json --noEmit", - "test": "npm run build && node --test test/tool-parity.test.mjs", + "test": "npm run build && node --test test/tool-parity.test.mjs test/structured-output.test.mjs", "prepublishOnly": "npm run build" }, "dependencies": { diff --git a/src/server.ts b/src/server.ts index 801edaa..ce46442 100644 --- a/src/server.ts +++ b/src/server.ts @@ -6,7 +6,12 @@ * top of the official `wellmarked` JavaScript SDK — the SDK owns auth, * transport, retries, typed errors, and polymorphic job polling, so this * layer only translates MCP tool calls into SDK calls and renders the - * results back as agent-readable text. + * results back to the agent. + * + * Every tool declares an `outputSchema` and returns `structuredContent`: + * typed JSON an agent reads fields off directly, instead of pattern-matching + * `status` or `12/50` out of a rendered sentence. The text rendering is still + * sent alongside it for clients that predate structured output — see `ok()`. * * Tools: * - extract — one URL → clean Markdown. @@ -38,20 +43,47 @@ import { import { SERVER_NAME, SERVER_VERSION } from "./version.js"; -/** Text-only tool result helpers. */ +/** + * Tool result helpers. + * + * Every success carries the payload TWICE, on purpose: + * - `structuredContent` — typed JSON. This is what an agent should read + * fields off (`status`, `completed`, `ok`, `tokensSaved`), instead of + * pattern-matching them out of a rendered sentence. + * - the text block — the same data rendered for humans and for clients that + * predate structured output. The spec asks a tool that returns structured + * content to also return a functionally equivalent unstructured copy, and + * dropping it would silently blank the tool on older clients. + * + * Errors stay text-only: the SDK skips output validation when `isError` is + * set, and a failure has no typed shape beyond the code already in the message. + */ type ToolResult = { content: { type: "text"; text: string }[]; + structuredContent?: Record; isError?: boolean; }; -function ok(text: string): ToolResult { - return { content: [{ type: "text", text }] }; +function ok(text: string, structuredContent: Record): ToolResult { + return { content: [{ type: "text", text }], structuredContent }; } function fail(text: string): ToolResult { return { content: [{ type: "text", text }], isError: true }; } +/** + * Render an SDK object exactly as it will travel on the wire. + * + * Two things this fixes, both of which would fail output validation if the SDK + * object were passed through as-is: `Date` fields (`createdAt`, `finishedAt`, + * `retrievedAt`) become ISO strings, and the SDK's computed getters (`ok`, + * `done`) materialize as plain boolean fields. + */ +function jsonify(value: unknown): Record { + return JSON.parse(JSON.stringify(value)) as Record; +} + /** * Translate any throwable from an SDK call into an `isError` tool result. * @@ -300,6 +332,129 @@ function renderUsage(u: Usage): string { ].join("\n"); } +// ── Output schemas ──────────────────────────────────────────────────────────── +// These mirror the SDK's response types (see `wellmarked`'s models.d.ts) and are +// published to clients as each tool's `outputSchema`, so an agent knows the +// shape of `structuredContent` before it ever calls the tool. +// +// They are deliberately a projection of the SDK types rather than a +// generated-from-source artifact: the SDK's types are TypeScript interfaces, +// which don't survive to runtime, and Zod is what the MCP SDK validates +// against. Drift is caught by test/structured-output.test.mjs, which asserts +// every tool's real result parses against its declared schema. + +/** Content fields shared by every extraction result. Exactly one is populated, + * per the request's `format`; the rest are null. */ +const contentOutputShape = { + markdown: z.string().nullable(), + blocks: z + .array( + z.object({ + type: z.enum(["heading", "paragraph", "list", "code"]), + text: z.string(), + level: z.number().int().nullable(), + }), + ) + .nullable(), + chunks: z + .array( + z.object({ + text: z.string(), + startToken: z.number().int(), + endToken: z.number().int(), + }), + ) + .nullable(), + html: z.string().nullable(), + links: z.array(z.string()).nullable(), + metrics: z + .object({ + contentBytes: z.number(), + inputTokens: z.number().nullable(), + outputTokens: z.number().nullable(), + tokensSaved: z.number().nullable(), + reductionPct: z.number().nullable(), + }) + .nullable(), +}; + +const metadataSchema = z.object({ + url: z.string(), + title: z.string().nullable(), + author: z.string().nullable(), + date: z.string().nullable(), + retrievedAt: z.string().nullable().describe("ISO 8601 timestamp."), +}); + +const extractOutputShape = { + ...contentOutputShape, + metadata: metadataSchema, + requestId: z.string(), +}; + +/** One bulk item or crawl page. `depth` is crawl-only, hence optional. */ +const jobItemSchema = z.object({ + ...contentOutputShape, + url: z.string(), + metadata: metadataSchema.nullable(), + error: z + .string() + .nullable() + .describe("Stable API error code (e.g. target_timeout), never a message."), + ok: z.boolean(), + depth: z.number().int().optional().describe("Crawl only: BFS depth from the root."), +}); + +/** One schema for both job kinds — `bulk`, `crawl`, `get_job`, and + * `wait_for_job` all return a BulkJob or a CrawlJob, and `get_job` is + * polymorphic by design. Crawl-only fields are optional; read `kind` to + * discriminate. */ +const jobOutputShape = { + kind: z.enum(["bulk", "crawl"]), + jobId: z.string(), + status: z.enum(["queued", "processing", "done"]), + total: z.number().int(), + completed: z.number().int(), + done: z.boolean().describe("True when status === 'done'."), + results: z.array(jobItemSchema), + createdAt: z.string().nullable().describe("ISO 8601 timestamp."), + finishedAt: z.string().nullable().describe("ISO 8601 timestamp."), + webhookSigningSecret: z + .string() + .nullable() + .describe("Shown once, on the submission that minted it. Null on polls."), + truncated: z.boolean().optional().describe("Crawl only."), + truncatedReason: z + .enum(["page_cap_reached", "quota_exhausted"]) + .nullable() + .optional() + .describe("Crawl only."), +}; + +const searchOutputShape = { + query: z.string(), + requestId: z.string(), + results: z.array( + z.object({ + ...contentOutputShape, + url: z.string(), + status: z.enum(["ok", "error"]), + title: z.string().nullable(), + snippet: z.string().nullable(), + error: z.string().nullable(), + ok: z.boolean(), + }), + ), +}; + +const usageOutputShape = { + plan: z.string(), + period: z.string(), + used: z.number(), + limit: z.number(), + remaining: z.number(), +}; + /** * Build a configured MCP server. The `WellMarked` client is created once and * shared across all tool invocations for the life of the process. @@ -339,6 +494,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { ...formatInputSchema, ...retryInputSchema, }, + outputSchema: extractOutputShape, annotations: { readOnlyHint: true, openWorldHint: true, @@ -353,7 +509,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { retry, ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }), }); - return ok(renderExtract(result)); + return ok(renderExtract(result), jsonify(result)); } catch (err) { return toErrorResult(err); } @@ -397,6 +553,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { ...formatInputSchema, ...retryInputSchema, }, + outputSchema: jobOutputShape, annotations: { openWorldHint: true }, }, async ({ urls, render_js, format, retry, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => { @@ -412,7 +569,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { timeoutMs: wait_timeout_ms ?? 300_000, })) as BulkJob; } - return ok(renderJob(job)); + return ok(renderJob(job), jsonify(job)); } catch (err) { return toErrorResult(err); } @@ -469,6 +626,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { ...formatInputSchema, ...retryInputSchema, }, + outputSchema: jobOutputShape, annotations: { openWorldHint: true }, }, async ({ url, depth, render_js, format, retry, max_pages, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => { @@ -486,7 +644,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { timeoutMs: wait_timeout_ms ?? 300_000, }); } - return ok(renderJob(job)); + return ok(renderJob(job), jsonify(job)); } catch (err) { return toErrorResult(err); } @@ -521,6 +679,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { ...policyInputSchema, ...formatInputSchema, }, + outputSchema: searchOutputShape, annotations: { readOnlyHint: true, openWorldHint: true, @@ -534,7 +693,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { format, ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }), }); - return ok(renderSearch(res)); + return ok(renderSearch(res), jsonify(res)); } catch (err) { return toErrorResult(err); } @@ -553,6 +712,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { inputSchema: { job_id: z.string().describe("The job id returned by bulk or crawl."), }, + outputSchema: jobOutputShape, annotations: { readOnlyHint: true, openWorldHint: true, @@ -562,7 +722,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { async ({ job_id }) => { try { const job = await client.getJob(job_id); - return ok(renderJob(job)); + return ok(renderJob(job), jsonify(job)); } catch (err) { return toErrorResult(err); } @@ -594,6 +754,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { .optional() .describe("Ms between status polls. Default 2000."), }, + outputSchema: jobOutputShape, annotations: { readOnlyHint: true, openWorldHint: true }, }, async ({ job_id, timeout_ms, poll_interval_ms }) => { @@ -602,7 +763,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { timeoutMs: timeout_ms ?? 300_000, pollIntervalMs: poll_interval_ms ?? 2_000, }); - return ok(renderJob(job)); + return ok(renderJob(job), jsonify(job)); } catch (err) { return toErrorResult(err); } @@ -620,6 +781,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { "the quota. Check this before a large bulk or crawl to avoid " + "hitting the monthly limit mid-job.", inputSchema: {}, + outputSchema: usageOutputShape, annotations: { readOnlyHint: true, openWorldHint: true, @@ -629,7 +791,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { async () => { try { const usage = await client.getUsage(); - return ok(renderUsage(usage)); + return ok(renderUsage(usage), jsonify(usage)); } catch (err) { return toErrorResult(err); } diff --git a/test/structured-output.test.mjs b/test/structured-output.test.mjs new file mode 100644 index 0000000..0c16d87 --- /dev/null +++ b/test/structured-output.test.mjs @@ -0,0 +1,314 @@ +// Structured tool output: every tool returns typed JSON, not prose to parse. +// +// The thing under test is the seam an agent actually consumes. Before this, +// `get_job` handed a model `status: running (12/50)` inside a sentence and the +// model had to pattern-match `status`, split `12/50`, and read `—` as null. +// Now the same call carries `structuredContent` with real JSON types. +// +// The stub sits at `globalThis.fetch`, NOT at the client methods, so each test +// exercises the whole chain: canned API JSON → the real SDK parser (snake_case +// → camelCase, Date coercion, the computed `ok`/`done` getters) → jsonify → +// the MCP SDK's own output validation against the declared outputSchema. +// That last step is why a passing call is itself proof of schema conformance: +// McpServer.validateToolOutput throws if structuredContent is missing or fails +// the schema, and the throw surfaces as a rejected callTool. +// +// Zero test deps, matching test/tool-parity.test.mjs: Node's built-in runner +// and the SDK's in-memory transport, against the built dist/. +import test from "node:test"; +import assert from "node:assert/strict"; + +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; + +import { createServer } from "../dist/server.js"; + +const DUMMY_KEY = "wm_" + "0".repeat(40); + +const META = { + url: "https://example.com/a", + title: "Hello", + author: "Ada Lovelace", + date: "2026-01-02", + retrieved_at: "2026-01-03T04:05:06Z", +}; + +const METRICS = { + content_bytes: 1024, + input_tokens: 5000, + output_tokens: 400, + tokens_saved: 4600, + reduction_pct: 92, +}; + +const EXTRACT_BODY = { + markdown: "# Hello\n\nBody text.", + metadata: META, + metrics: METRICS, + request_id: "req_123", +}; + +const BULK_BODY = { + kind: "bulk", + job_id: "job_abc", + status: "done", + total: 2, + completed: 2, + created_at: "2026-01-03T04:00:00Z", + finished_at: "2026-01-03T04:00:30Z", + results: [ + { url: "https://example.com/a", markdown: "# A", metadata: META, error: null }, + { url: "https://example.com/b", markdown: null, metadata: null, error: "target_timeout" }, + ], +}; + +const CRAWL_BODY = { + kind: "crawl", + job_id: "job_crawl", + status: "done", + total: 1, + completed: 1, + truncated: true, + truncated_reason: "page_cap_reached", + created_at: "2026-01-03T04:00:00Z", + finished_at: "2026-01-03T04:01:00Z", + results: [ + { url: "https://example.com/a", depth: 1, markdown: "# A", metadata: META, error: null }, + ], +}; + +const SEARCH_BODY = { + query: "ada lovelace", + request_id: "req_s1", + results: [ + { + url: "https://example.com/a", + status: "ok", + title: "Hello", + snippet: "a snippet", + markdown: "# A", + metrics: METRICS, + }, + { + url: "https://example.com/b", + status: "error", + title: null, + snippet: "another snippet", + error: "target_timeout", + }, + ], +}; + +const USAGE_BODY = { plan: "pro", period: "2026-01", used: 120, limit: 10000, remaining: 9880 }; + +/** Point globalThis.fetch at canned bodies keyed by "METHOD /path". */ +function stubFetch(routes) { + const previous = globalThis.fetch; + globalThis.fetch = async (url, init = {}) => { + const method = (init.method || "GET").toUpperCase(); + const { pathname } = new URL(String(url)); + const key = `${method} ${pathname}`; + const entry = routes[key]; + if (entry === undefined) { + throw new Error(`unstubbed request: ${key}`); + } + const { status = 200, body } = entry.body === undefined ? { body: entry } : entry; + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + }; + return () => { + globalThis.fetch = previous; + }; +} + +/** Call one tool over a real client↔server transport pair. */ +async function callTool(name, args = {}) { + const server = createServer({ apiKey: DUMMY_KEY }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "structured-output-test", version: "0.0.0" }); + await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); + try { + return await client.callTool({ name, arguments: args }); + } finally { + await client.close(); + await server.close(); + } +} + +async function listTools() { + const server = createServer({ apiKey: DUMMY_KEY }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "structured-output-test", version: "0.0.0" }); + await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); + const { tools } = await client.listTools(); + await client.close(); + await server.close(); + return tools; +} + +// ── Discovery ──────────────────────────────────────────────────────────────── + +test("every tool advertises an outputSchema", async () => { + const tools = await listTools(); + const missing = tools.filter((t) => !t.outputSchema).map((t) => t.name); + assert.deepEqual(missing, [], `tools without an outputSchema: ${missing.join(", ")}`); + assert.equal(tools.length, 7); +}); + +test("the advertised job schema types the fields an agent branches on", async () => { + const tools = await listTools(); + const props = tools.find((t) => t.name === "get_job").outputSchema.properties; + assert.equal(props.status.type, "string"); + assert.deepEqual(props.status.enum, ["queued", "processing", "done"]); + assert.equal(props.completed.type, "integer"); + assert.equal(props.total.type, "integer"); + assert.equal(props.done.type, "boolean"); +}); + +// ── Payloads ───────────────────────────────────────────────────────────────── + +test("extract returns typed structured content alongside the text block", async () => { + const restore = stubFetch({ "POST /extract": EXTRACT_BODY }); + try { + const res = await callTool("extract", { url: "https://example.com/a" }); + const sc = res.structuredContent; + + assert.ok(sc, "extract must return structuredContent"); + assert.equal(sc.requestId, "req_123"); + assert.equal(sc.markdown, "# Hello\n\nBody text."); + assert.equal(sc.metadata.title, "Hello"); + assert.equal(sc.metadata.author, "Ada Lovelace"); + // Date -> ISO string, not a serialized Date object or an em dash. + assert.equal(sc.metadata.retrievedAt, "2026-01-03T04:05:06.000Z"); + // Metrics arrive as numbers, not "saved 4600, -92%" inside a sentence. + assert.equal(sc.metrics.tokensSaved, 4600); + assert.equal(typeof sc.metrics.reductionPct, "number"); + + // Back-compat: clients that ignore structuredContent still get the render. + assert.equal(res.content[0].type, "text"); + assert.match(res.content[0].text, /Body text\./); + assert.notEqual(res.isError, true); + } finally { + restore(); + } +}); + +test("get_job returns job state as real JSON types, not prose", async () => { + const restore = stubFetch({ "GET /bulk/job_abc": BULK_BODY }); + try { + const res = await callTool("get_job", { job_id: "job_abc" }); + const sc = res.structuredContent; + + assert.equal(sc.kind, "bulk"); + assert.equal(sc.jobId, "job_abc"); + assert.equal(sc.status, "done"); + // The three fields that previously only existed as "(2/2)" in a sentence. + assert.equal(sc.completed, 2); + assert.equal(sc.total, 2); + assert.equal(sc.done, true); + assert.equal(typeof sc.completed, "number"); + assert.equal(typeof sc.done, "boolean"); + assert.equal(sc.createdAt, "2026-01-03T04:00:00.000Z"); + + // Per-item success is a boolean, not a ✓/✗ glyph. + assert.equal(sc.results.length, 2); + assert.equal(sc.results[0].ok, true); + assert.equal(sc.results[1].ok, false); + assert.equal(sc.results[1].error, "target_timeout"); + } finally { + restore(); + } +}); + +test("a finished job with no results still validates", async () => { + // The empty-results branch renders "(no results)" as text; the structured + // side must stay a real empty array rather than that string. + const restore = stubFetch({ + "GET /bulk/job_empty": { ...BULK_BODY, job_id: "job_empty", total: 0, completed: 0, results: [] }, + }); + try { + const res = await callTool("get_job", { job_id: "job_empty" }); + assert.deepEqual(res.structuredContent.results, []); + assert.equal(res.structuredContent.total, 0); + } finally { + restore(); + } +}); + +test("crawl carries its crawl-only fields through the shared job schema", async () => { + const restore = stubFetch({ + "POST /crawl": CRAWL_BODY, + "GET /crawl/job_crawl": CRAWL_BODY, + }); + try { + const res = await callTool("crawl", { url: "https://example.com", depth: 1 }); + const sc = res.structuredContent; + + assert.equal(sc.kind, "crawl"); + assert.equal(sc.truncated, true); + assert.equal(sc.truncatedReason, "page_cap_reached"); + assert.equal(sc.results[0].depth, 1); + } finally { + restore(); + } +}); + +test("search returns per-result status as a boolean", async () => { + const restore = stubFetch({ "POST /search": SEARCH_BODY }); + try { + const res = await callTool("search", { query: "ada lovelace" }); + const sc = res.structuredContent; + + assert.equal(sc.query, "ada lovelace"); + assert.equal(sc.requestId, "req_s1"); + assert.equal(sc.results[0].ok, true); + assert.equal(sc.results[1].ok, false); + assert.equal(sc.results[1].error, "target_timeout"); + assert.equal(sc.results[0].metrics.tokensSaved, 4600); + } finally { + restore(); + } +}); + +test("get_usage returns numbers, not a formatted percentage line", async () => { + const restore = stubFetch({ "GET /usage": USAGE_BODY }); + try { + const res = await callTool("get_usage", {}); + const sc = res.structuredContent; + + assert.deepEqual(sc, { + plan: "pro", + period: "2026-01", + used: 120, + limit: 10000, + remaining: 9880, + }); + } finally { + restore(); + } +}); + +// ── Error path ─────────────────────────────────────────────────────────────── + +test("an API error stays text-only and does not trip output validation", async () => { + // Declaring an outputSchema makes structuredContent mandatory on success. + // Errors are exempt (McpServer.validateToolOutput returns early on isError), + // and this pins that: if the exemption ever stopped applying, the tool would + // reject with an output-validation McpError instead of returning isError. + const restore = stubFetch({ + "POST /extract": { + status: 422, + body: { error: { code: "unsafe_url", message: "URL resolves to a private address." } }, + }, + }); + try { + const res = await callTool("extract", { url: "https://example.com/a" }); + assert.equal(res.isError, true); + assert.equal(res.structuredContent, undefined); + assert.match(res.content[0].text, /unsafe_url/); + } finally { + restore(); + } +});