diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ba8ca8..2834171 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,11 @@ jobs: cache: npm # Resolves "wellmarked" from npm (the published JS/TS SDK), not a # local file: path — so this builds in an isolated repo checkout. + # NOTE: a build/test here fails until the JS SDK is published with any + # new surface this server uses (e.g. the `search` tool needs the SDK's + # `search()`), which is the correct gate — publish the SDK first. - run: npm ci - run: npm run typecheck - - run: npm run build + # `npm test` builds (rimraf + tsc) then runs the stdio/HTTP tool-parity + # test, so it covers the deployable dist AND the continuity guarantee. + - run: npm test diff --git a/.gitignore b/.gitignore index 397b0d9..fa75087 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist/ *.log .env .DS_Store +.idea/ diff --git a/README.md b/README.md index a94180b..f4f9828 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,35 @@ hints, and polymorphic job polling. ## Setup -You need a WellMarked API key (`wm_...`) — generate one at -[wellmarked.io](https://wellmarked.io). +There are two ways to connect, depending on your host: + +- **Remote (hosted, OAuth)** — for Claude.ai and other hosts that support + remote MCP servers. No API key to copy; you sign in and authorize in the + browser. See [Remote server](#remote-server-hosted--oauth) below. +- **Local (stdio, API key)** — for Claude Desktop, Claude Code, Cursor, and any + host that launches a local server. You supply a `wm_...` key via the + environment. Generate one at [wellmarked.io](https://wellmarked.io). + +### Remote server (hosted — OAuth) + +Add WellMarked as a **custom connector** and point it at: + +``` +https://mcp.wellmarked.io/mcp +``` + +Your host discovers the authorization server automatically (via +`/.well-known/oauth-protected-resource`), walks you through signing in to +WellMarked and approving the connection, and receives a scoped, expiring token — +no key to paste. The connector can `extract`, `bulk`, and `crawl`; it cannot +mint or rotate credentials. + +In **Claude.ai**: Settings → Connectors → Add custom connector → paste the URL +above → follow the sign-in and consent prompts. + +The remote server speaks MCP over Streamable HTTP and, under the hood, calls the +same tools as the local server — the tool surface is identical (enforced by the +parity test in `test/`). ### Claude Desktop @@ -61,20 +88,34 @@ Any host that launches MCP servers over stdio works — point it at ## Environment variables +Local (stdio) server: + | Variable | Required | Description | | --- | --- | --- | | `WELLMARKED_API_KEY` | yes | Your `wm_...` API key. | -| `WELLMARKED_BASE_URL` | no | Override the API base URL (self-hosted instances). | | `WELLMARKED_TIMEOUT_MS` | no | Per-request timeout in ms (default `30000`). | +Remote (Streamable HTTP) server — only if you self-host it (`npm run start:http`): + +| Variable | Required | Description | +| --- | --- | --- | +| `PORT` | no | Listen port (default `3000`; Railway sets it). | +| `MCP_PUBLIC_URL` | no | This server's public URL, used in the protected-resource metadata. Derived from the request Host if unset. | +| `WELLMARKED_TIMEOUT_MS` | no | Per-request timeout in ms. | + +The remote server takes **no** `WELLMARKED_API_KEY` — each request authenticates +with its own OAuth bearer token. + ## Develop locally ```bash npm install npm run build # compile TypeScript to dist/ npm start # run the compiled server on stdio +npm run start:http # OR run the remote (Streamable HTTP) server +npm test # tool-list parity across both entry points -# Inspect it interactively with the MCP Inspector: +# Inspect the stdio server interactively with the MCP Inspector: WELLMARKED_API_KEY=wm_... npx @modelcontextprotocol/inspector node dist/index.js ``` diff --git a/package.json b/package.json index fb341f6..6e38db7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wellmarked-mcp", - "version": "1.0.0", + "version": "1.1.0", "description": "Official Model Context Protocol (MCP) server for the WellMarked API — let AI agents convert any URL to clean Markdown, bulk-extract, and crawl sites.", "keywords": [ "wellmarked", @@ -44,8 +44,10 @@ "scripts": { "build": "rimraf dist && tsc -p tsconfig.json", "start": "node dist/index.js", + "start:http": "node dist/http.js", "dev": "tsc -p tsconfig.json --watch", "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "npm run build && node --test \"test/**/*.test.mjs\"", "prepublishOnly": "npm run build" }, "dependencies": { diff --git a/railway.toml b/railway.toml new file mode 100644 index 0000000..86fabcd --- /dev/null +++ b/railway.toml @@ -0,0 +1,18 @@ +# The remote MCP server runs as its OWN Railway service (Phase 7) — separate +# from the API and website. It's a stateless Node process; Nixpacks detects the +# package.json, installs, and runs `npm run build`, then the startCommand below. +# +# The default `npm start` is the STDIO entry point (dist/index.js) for local +# hosts — wrong for a hosted service — so the startCommand overrides it with the +# Streamable HTTP entry point. Railway injects PORT; http.ts reads it. +# +# Required env: none. Optional: WELLMARKED_BASE_URL (defaults to the public API), +# MCP_PUBLIC_URL (defaults to the request Host), WELLMARKED_TIMEOUT_MS. +[build] +builder = "NIXPACKS" + +[deploy] +startCommand = "npm run start:http" +healthcheckPath = "/health" +healthcheckTimeout = 30 +restartPolicyType = "on_failure" diff --git a/src/http.ts b/src/http.ts new file mode 100644 index 0000000..6e1fa12 --- /dev/null +++ b/src/http.ts @@ -0,0 +1,219 @@ +#!/usr/bin/env node +/** + * Remote entry point for the WellMarked MCP server — Streamable HTTP + OAuth + * (Phase 7). + * + * A sibling to index.ts (stdio). It speaks MCP over HTTP so hosted clients + * (Claude.ai custom connectors, etc.) can reach it without launching a local + * process, and it authenticates with OAuth 2.1 bearer tokens instead of an + * env-var API key. + * + * The seam that makes this small: it calls the SAME `createServer` as the + * stdio entry point, once per request, with the request's own bearer token as + * the API key. No tool code changes — the tool surface is transport-independent + * (see the parity test in test/). + * + * OAuth roles (RFC 9728 / the MCP authorization spec): + * - This process is the RESOURCE SERVER. It publishes + * `/.well-known/oauth-protected-resource` pointing at the WellMarked API as + * the authorization server, and rejects unauthenticated/expired requests + * with `401 WWW-Authenticate: Bearer resource_metadata=...` so the client + * knows to run (or refresh) the OAuth flow. + * - The WellMarked API is the AUTHORIZATION SERVER (api/routes/oauth.py). + * - An access token is just a `wm_...` key with an expiry, so validation is a + * single authenticated probe to the API — no bespoke token introspection. + * + * Runs STATELESS: a fresh transport + server per request. Access tokens expire + * hourly, so binding one to a long-lived session would break mid-session; + * per-request binding always uses the token the client just presented, and our + * tools are all request/response (no server-initiated streaming to preserve). + * + * Environment: + * - MCP_PUBLIC_URL (optional) — this server's public URL, used in the + * protected-resource metadata. Derived from the + * request Host if unset. + * - PORT (optional) — listen port. Default 3000 (Railway sets it). + * - WELLMARKED_TIMEOUT_MS (optional) — per-request timeout in ms. + */ +import { createServer as createHttpServer, type IncomingMessage, type ServerResponse } from "node:http"; + +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import type { AuthInfo } from "@modelcontextprotocol/sdk/server/auth/types.js"; + +import { createServer } from "./server.js"; + +const API_BASE_URL = "https://api.wellmarked.io"; +const PORT = Number.parseInt(process.env.PORT || "3000", 10); + +// The scopes an OAuth connector token carries — the MCP tool surface. Mirrors +// api/services/oauth.OAUTH_SCOPES. Advertised in the protected-resource +// metadata; the API is the one that actually enforces them per tool. +const OAUTH_SCOPES = ["extract", "bulk", "crawl"]; + +function resolveTimeout(): number | undefined { + const raw = process.env.WELLMARKED_TIMEOUT_MS; + if (!raw) return undefined; + const n = Number.parseInt(raw, 10); + return Number.isFinite(n) && n > 0 ? n : undefined; +} +const TIMEOUT_MS = resolveTimeout(); + +function publicBaseUrl(req: IncomingMessage): string { + if (process.env.MCP_PUBLIC_URL) return process.env.MCP_PUBLIC_URL.replace(/\/+$/, ""); + const proto = String(req.headers["x-forwarded-proto"] || "https").split(",")[0].trim(); + const host = req.headers.host || `localhost:${PORT}`; + return `${proto}://${host}`; +} + +function resourceMetadataUrl(req: IncomingMessage): string { + return `${publicBaseUrl(req)}/.well-known/oauth-protected-resource`; +} + +function setCors(res: ServerResponse): void { + // Tokens are bearer, not cookies, so a wildcard origin is safe (no + // credentialed CORS). Expose WWW-Authenticate so a browser client can read + // the resource_metadata hint that starts the OAuth flow. + res.setHeader("Access-Control-Allow-Origin", "*"); + res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + res.setHeader("Access-Control-Allow-Headers", "Authorization, Content-Type, Mcp-Session-Id, Mcp-Protocol-Version"); + res.setHeader("Access-Control-Expose-Headers", "WWW-Authenticate, Mcp-Session-Id"); +} + +function sendJson(res: ServerResponse, status: number, body: unknown): void { + const payload = JSON.stringify(body); + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(payload); +} + +function unauthorized(req: IncomingMessage, res: ServerResponse, invalid: boolean): void { + // RFC 6750 / MCP: point the client at the protected-resource metadata so it + // can discover the authorization server and (re)authorize. `invalid_token` + // distinguishes an expired/revoked token (refresh) from a missing one. + const params = [`resource_metadata="${resourceMetadataUrl(req)}"`]; + if (invalid) params.unshift(`error="invalid_token"`); + res.setHeader("WWW-Authenticate", `Bearer ${params.join(", ")}`); + sendJson(res, 401, { + error: invalid ? "invalid_token" : "missing_token", + error_description: invalid + ? "The access token is invalid or expired." + : "Authorization required. Provide a Bearer access token.", + }); +} + +/** + * Validate a bearer token against the WellMarked API. An access token is a + * `wm_...` key, so a single authenticated probe to an unmetered, no-scope + * endpoint (`/usage`) tells us validity: 200 → valid, 401/403 → invalid/expired, + * anything else (or a network failure) → the API is unavailable (fail closed). + */ +async function verifyToken(token: string): Promise<"valid" | "invalid" | "unavailable"> { + try { + const resp = await fetch(`${API_BASE_URL}/usage`, { + headers: { Authorization: `Bearer ${token}` }, + }); + if (resp.status === 200) return "valid"; + if (resp.status === 401 || resp.status === 403) return "invalid"; + return "unavailable"; + } catch { + return "unavailable"; + } +} + +function readBody(req: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + if (!raw) return resolve(undefined); + try { + resolve(JSON.parse(raw)); + } catch (e) { + reject(e); + } + }); + req.on("error", reject); + }); +} + +async function handleMcp(req: IncomingMessage, res: ServerResponse): Promise { + const header = req.headers.authorization || ""; + const token = header.startsWith("Bearer ") ? header.slice("Bearer ".length).trim() : ""; + if (!token) return unauthorized(req, res, false); + + const verdict = await verifyToken(token); + if (verdict === "invalid") return unauthorized(req, res, true); + if (verdict === "unavailable") { + return sendJson(res, 503, { + error: "service_unavailable", + error_description: "Unable to reach the WellMarked API to validate the token.", + }); + } + + let body: unknown; + try { + body = await readBody(req); + } catch { + // JSON-RPC parse error. + return sendJson(res, 400, { jsonrpc: "2.0", error: { code: -32700, message: "Parse error" }, id: null }); + } + + // Stateless: one transport + server per request, keyed on this request's token. + const server = createServer({ apiKey: token, timeoutMs: TIMEOUT_MS }); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined }); + res.on("close", () => { + void transport.close(); + void server.close(); + }); + await server.connect(transport); + + const auth: AuthInfo = { token, clientId: "wellmarked-mcp", scopes: OAUTH_SCOPES }; + const authedReq = req as IncomingMessage & { auth?: AuthInfo }; + authedReq.auth = auth; + await transport.handleRequest(authedReq, res, body); +} + +const httpServer = createHttpServer((req, res) => { + setCors(res); + if (req.method === "OPTIONS") { + res.writeHead(204); + res.end(); + return; + } + + const url = new URL(req.url || "/", "http://localhost"); + + if (req.method === "GET" && url.pathname === "/health") { + return sendJson(res, 200, { status: "ok" }); + } + + if (req.method === "GET" && url.pathname === "/.well-known/oauth-protected-resource") { + return sendJson(res, 200, { + resource: `${publicBaseUrl(req)}/mcp`, + authorization_servers: [API_BASE_URL], + scopes_supported: OAUTH_SCOPES, + bearer_methods_supported: ["header"], + }); + } + + if (url.pathname === "/mcp") { + if (req.method === "POST") { + void handleMcp(req, res).catch((err) => { + process.stderr.write(`[wellmarked-mcp] request failed: ${err instanceof Error ? err.message : String(err)}\n`); + if (!res.headersSent) { + sendJson(res, 500, { jsonrpc: "2.0", error: { code: -32603, message: "Internal error" }, id: null }); + } + }); + return; + } + // Stateless mode has no standalone SSE stream or session teardown. + res.setHeader("Allow", "POST"); + return sendJson(res, 405, { error: "method_not_allowed" }); + } + + sendJson(res, 404, { error: "not_found" }); +}); + +httpServer.listen(PORT, () => { + process.stderr.write(`[wellmarked-mcp] Streamable HTTP listening on :${PORT} (API ${API_BASE_URL})\n`); +}); diff --git a/src/index.ts b/src/index.ts index f35d121..cf83064 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,7 +12,6 @@ * * Environment: * - WELLMARKED_API_KEY (required) — your `wm_...` key. - * - WELLMARKED_BASE_URL (optional) — override the API base URL. * - WELLMARKED_TIMEOUT_MS(optional) — per-request timeout in ms. */ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; @@ -31,7 +30,6 @@ async function main(): Promise { try { server = createServer({ apiKey: process.env.WELLMARKED_API_KEY, - baseUrl: process.env.WELLMARKED_BASE_URL, timeoutMs: resolveTimeout(), }); } catch (err) { diff --git a/src/server.ts b/src/server.ts index cc50211..801edaa 100644 --- a/src/server.ts +++ b/src/server.ts @@ -29,8 +29,10 @@ import { WellMarkedError, type WellMarkedOptions, type BulkJob, + type Content, type CrawlJob, type ExtractResult, + type SearchResults, type Usage, } from "wellmarked"; @@ -80,6 +82,126 @@ function iso(d: Date | null): string { return d ? d.toISOString() : "—"; } +// ── Compliance overrides ──────────────────────────────────────────────────── +// Shared across extract / bulk / crawl. These can only NARROW the API key's own +// policy server-side (add denies, restrict domains, upgrade robots to strict), +// never widen it — so exposing them to an agent is safe. +const policyInputSchema = { + allow_domains: z + .array(z.string()) + .optional() + .describe( + "Restrict this request to these domains (and their subdomains). Can only " + + "narrow the key's own allow-list, never widen it.", + ), + deny_patterns: z + .array(z.string()) + .optional() + .describe( + "Extra deny globs for this request, matched against the hostname and the " + + "full URL. Added to the key's deny list.", + ), + respect_robots: z + .enum(["strict", "lax"]) + .optional() + .describe( + "'strict' honors robots.txt on this request (extract/bulk too, not just " + + "crawl); can tighten the key's setting but never loosen it.", + ), +}; + +// ── Output format ─────────────────────────────────────────────────────────── +// Shared across extract / bulk / crawl. Kept off `search`, which always returns +// markdown — search hands an agent N pages to read, and the other formats serve +// pipelines that already know which URL they want. +const formatInputSchema = { + format: z + .enum(["markdown", "json", "chunks", "html", "links"]) + .optional() + .describe( + "Output format. 'markdown' (default) is clean prose; 'json' returns typed " + + "heading/paragraph/list/code blocks; 'chunks' returns contiguous " + + "500-token windows ready for embedding; 'html' returns the raw fetched " + + "HTML; 'links' returns every http(s) link on the page. 'json' and " + + "'chunks' require a Pro, Growth, or Enterprise plan.", + ), +}; + +// ── Timeout retries ───────────────────────────────────────────────────────── +// Shared across extract / bulk / crawl. Kept off `search`: its 15s per-result +// deadline can't absorb a retry, so the API doesn't take one there. +const retryInputSchema = { + retry: z + .number() + .int() + .min(0) + .optional() + .describe( + "Server-side re-attempts when the target times out, each on a fresh " + + "connection. Default 0 (one attempt). On the synchronous extract " + + "each timed-out attempt takes 20-30s, so prefer bulk for aggressive " + + "values.", + ), +}; + +/** + * Render whichever content field a result carries into text. + * + * MCP tools return text, so each non-markdown format needs a legible textual + * projection — an agent reading `[object Object]` is strictly worse off than + * one reading markdown. Chunks and blocks keep their structure visible + * (offsets, block types) because that's the reason the caller asked for them. + */ +function renderContent(item: Content): string | null { + if (item.markdown !== null) return item.markdown; + if (item.html !== null) return item.html; + if (item.links !== null) return item.links.join("\n"); + if (item.blocks !== null) { + return item.blocks + .map((b) => { + const label = b.type === "heading" ? `heading h${b.level ?? 1}` : b.type; + return `[${label}] ${b.text}`; + }) + .join("\n\n"); + } + if (item.chunks !== null) { + return item.chunks + .map((c) => `[tokens ${c.startToken}-${c.endToken}]\n${c.text}`) + .join("\n\n"); + } + return null; +} + +/** One line of ROI metrics, when the API reported them. */ +function renderMetrics(item: Content): string | null { + const m = item.metrics; + if (!m || m.tokensSaved === null) return null; + return ( + `Tokens: ${m.outputTokens} out of ${m.inputTokens} raw ` + + `(saved ${m.tokensSaved}, −${m.reductionPct}%)` + ); +} + +interface PolicyToolArgs { + allow_domains?: string[]; + deny_patterns?: string[]; + respect_robots?: "strict" | "lax"; +} + +/** Map the snake_case tool args to the SDK's camelCase policy options. Unset + * fields stay undefined; the SDK omits them so the key's policy is untouched. */ +function toPolicyOptions(a: PolicyToolArgs): { + allowDomains?: string[]; + denyPatterns?: string[]; + respectRobots?: "strict" | "lax"; +} { + return { + allowDomains: a.allow_domains, + denyPatterns: a.deny_patterns, + respectRobots: a.respect_robots, + }; +} + // ── Renderers ───────────────────────────────────────────────────────────────── // Render SDK objects as text an LLM can consume directly: the Markdown is the // payload the agent actually wants, so it goes in verbatim, with a compact @@ -97,7 +219,9 @@ function renderExtract(r: ExtractResult): string { ] .filter(Boolean) .join("\n"); - return `${header}\n\n---\n\n${r.markdown}`; + const metrics = renderMetrics(r); + const full = metrics ? `${header}\n${metrics}` : header; + return `${full}\n\n---\n\n${renderContent(r) ?? "(no content)"}`; } function renderJob(job: BulkJob | CrawlJob): string { @@ -133,9 +257,34 @@ function renderJob(job: BulkJob | CrawlJob): string { "depth" in item ? ` (depth ${(item as { depth: number }).depth})` : ""; const flag = item.ok ? "✓" : `✗ error: ${item.error ?? "unknown"}`; lines.push(`## ${item.url}${depth} ${flag}`); - if (item.ok && item.markdown) { + const body = item.ok ? renderContent(item) : null; + if (body) { lines.push(""); - lines.push(item.markdown); + lines.push(body); + } + } + return lines.join("\n"); +} + +function renderSearch(res: SearchResults): string { + const lines: string[] = [ + `Search: ${res.query} (${res.results.length} results)`, + `Request ID: ${res.requestId}`, + ]; + for (const item of res.results) { + lines.push(""); + const flag = item.ok ? "✓" : `✗ error: ${item.error ?? "unknown"}`; + lines.push(`## ${item.title || item.url} ${flag}`); + lines.push(item.url); + if (item.snippet) lines.push(`> ${item.snippet}`); + if (item.ok) { + // Format-agnostic: search carries the same format param as extract, so + // a chunks/json/html/links result renders through the shared path. + const content = renderContent(item); + if (content) { + lines.push(""); + lines.push(content); + } } } return lines.join("\n"); @@ -186,6 +335,9 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { "Needed for SPA / client-rendered pages. Requires a Pro+ plan " + "and the feature enabled on the API instance. Default false.", ), + ...policyInputSchema, + ...formatInputSchema, + ...retryInputSchema, }, annotations: { readOnlyHint: true, @@ -193,9 +345,14 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { idempotentHint: true, }, }, - async ({ url, render_js }) => { + async ({ url, render_js, format, retry, allow_domains, deny_patterns, respect_robots }) => { try { - const result = await client.extract(url, { renderJs: render_js }); + const result = await client.extract(url, { + renderJs: render_js, + format, + retry, + ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }), + }); return ok(renderExtract(result)); } catch (err) { return toErrorResult(err); @@ -236,12 +393,20 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { .positive() .optional() .describe("Max ms to wait when wait=true. Default 300000 (5 min)."), + ...policyInputSchema, + ...formatInputSchema, + ...retryInputSchema, }, annotations: { openWorldHint: true }, }, - async ({ urls, render_js, wait, wait_timeout_ms }) => { + async ({ urls, render_js, format, retry, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => { try { - let job = await client.bulk(urls, { renderJs: render_js }); + let job = await client.bulk(urls, { + renderJs: render_js, + format, + retry, + ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }), + }); if (wait !== false) { job = (await client.waitForJob(job.jobId, { timeoutMs: wait_timeout_ms ?? 300_000, @@ -291,14 +456,30 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { .positive() .optional() .describe("Max ms to wait when wait=true. Default 300000 (5 min)."), + max_pages: z + .number() + .int() + .min(1) + .optional() + .describe( + "Stop the crawl after this many successful pages. Can only " + + "narrow the plan's page cap, never widen it.", + ), + ...policyInputSchema, + ...formatInputSchema, + ...retryInputSchema, }, annotations: { openWorldHint: true }, }, - async ({ url, depth, render_js, wait, wait_timeout_ms }) => { + async ({ url, depth, render_js, format, retry, max_pages, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => { try { let job: BulkJob | CrawlJob = await client.crawl(url, { depth, renderJs: render_js, + format, + retry, + maxPages: max_pages, + ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }), }); if (wait === true) { job = await client.waitForJob(job.jobId, { @@ -312,6 +493,54 @@ export function createServer(options: WellMarkedOptions = {}): McpServer { }, ); + // ── search ───────────────────────────────────────────────────────────────── + server.registerTool( + "search", + { + title: "Search the web and extract the results", + description: + "Search the web for a query and get each result page back as clean " + + "Markdown — search plus extraction in one synchronous call, no job to " + + "poll. Use this to answer a question from the live web when you don't " + + "already have the URLs. Returns up to num_results pages, each with a " + + "status (a slow or blocked page comes back as an error item, not a " + + "failure of the whole call). Requires a Pro+ plan.", + inputSchema: { + query: z.string().min(1).describe("The search query."), + num_results: z + .number() + .int() + .min(1) + .max(10) + .optional() + .describe("How many results to fetch + extract (1–10). Default 5."), + render_js: z + .boolean() + .optional() + .describe("Render JavaScript on each result page before extracting. Default false."), + ...policyInputSchema, + ...formatInputSchema, + }, + annotations: { + readOnlyHint: true, + openWorldHint: true, + }, + }, + async ({ query, num_results, render_js, format, allow_domains, deny_patterns, respect_robots }) => { + try { + const res = await client.search(query, { + numResults: num_results, + renderJs: render_js, + format, + ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }), + }); + return ok(renderSearch(res)); + } catch (err) { + return toErrorResult(err); + } + }, + ); + // ── get_job ────────────────────────────────────────────────────────────── server.registerTool( "get_job", diff --git a/src/version.ts b/src/version.ts index e0fde21..030e623 100644 --- a/src/version.ts +++ b/src/version.ts @@ -3,4 +3,4 @@ * Keep SERVER_VERSION in sync with the `version` field in package.json. */ export const SERVER_NAME = "wellmarked-mcp"; -export const SERVER_VERSION = "1.0.0"; +export const SERVER_VERSION = "1.1.0"; diff --git a/test/tool-parity.test.mjs b/test/tool-parity.test.mjs new file mode 100644 index 0000000..f3af393 --- /dev/null +++ b/test/tool-parity.test.mjs @@ -0,0 +1,50 @@ +// Tool-list parity across transports (Phase 7, step 4). +// +// The stdio entry point (src/index.ts) and the remote HTTP entry point +// (src/http.ts) both build their McpServer from the SAME createServer(). The +// whole point of that shared seam is that an agent sees the identical tools +// whether it connects locally or over the network. This test pins that: it +// lists the tools through a real client↔server transport pair and asserts the +// exact surface. If a future edit forks tool definitions per transport, or +// drops/renames a tool, this fails. +// +// Zero test deps on purpose (the repo had no runner): Node's built-in test +// runner + the SDK's in-memory transport, against the built dist/. Run with +// `npm test` (builds first). +import test from "node:test"; +import assert from "node:assert/strict"; + +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; + +import { createServer } from "../dist/server.js"; + +const EXPECTED_TOOLS = ["bulk", "crawl", "extract", "get_job", "get_usage", "search", "wait_for_job"]; + +// A well-formed dummy key: createServer builds a WellMarked client but makes no +// network call, and listing tools is local (they're statically registered). +const DUMMY_KEY = "wm_" + "0".repeat(40); + +async function toolNames() { + const server = createServer({ apiKey: DUMMY_KEY }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "parity-test", version: "0.0.0" }); + await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); + const { tools } = await client.listTools(); + await client.close(); + await server.close(); + return tools.map((t) => t.name).sort(); +} + +test("the shared createServer exposes exactly the expected tool set", async () => { + const names = await toolNames(); + assert.deepEqual(names, EXPECTED_TOOLS); +}); + +test("two independent createServer() instances (stdio vs HTTP) are identical", async () => { + // index.ts and http.ts each call createServer() independently. Comparing two + // fresh instances is a faithful stand-in for "stdio transport tools" vs + // "HTTP transport tools": same surface, transport-independent. + const [a, b] = await Promise.all([toolNames(), toolNames()]); + assert.deepEqual(a, b); +});