From 9b3fc7dea6c009c289eee57ee177d722e1a4981b Mon Sep 17 00:00:00 2001
From: Grady Dillon <gradydillon22@gmail.com>
Date: Mon, 20 Jul 2026 16:45:25 -0400
Subject: [PATCH 1/6] Add remote (Streamable HTTP + OAuth) server, search tool,
 and tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- src/http.ts: a sibling to the stdio entry point that speaks MCP over
  Streamable HTTP, so hosted clients can connect without launching a
  local process. It calls the SAME createServer per request, so the tool
  surface is identical across transports. Dependency-free (raw node
  http). Runs stateless: access tokens expire hourly, so binding one to
  a long-lived session would break mid-session; per-request binding
  always uses the token just presented.
- OAuth resource-server role: serves /.well-known/oauth-protected-
  resource and answers unauthenticated or expired requests with 401 +
  WWW-Authenticate, which is what makes a client run (or refresh) the
  flow. A token is validated by one probe to the API, since an access
  token is just an API key; unreachable API fails closed with 503.
- search tool, mirroring the API's /search.
- test/: a zero-dependency (node:test) parity test asserting the stdio
  and HTTP entry points expose an identical tool list. CI now runs it —
  previously the repo had no test step at all.
- railway.toml to deploy the remote server as its own service.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 .github/workflows/ci.yml  |   7 +-
 .gitignore                |   1 +
 README.md                 |  49 ++++++++-
 package.json              |   2 +
 railway.toml              |  18 ++++
 src/http.ts               | 220 ++++++++++++++++++++++++++++++++++++++
 src/server.ts             | 132 ++++++++++++++++++++++-
 test/tool-parity.test.mjs |  50 +++++++++
 8 files changed, 470 insertions(+), 9 deletions(-)
 create mode 100644 railway.toml
 create mode 100644 src/http.ts
 create mode 100644 test/tool-parity.test.mjs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8ba8ca8..2834171 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,6 +24,11 @@ jobs:
           cache: npm
       # Resolves "wellmarked" from npm (the published JS/TS SDK), not a
       # local file: path — so this builds in an isolated repo checkout.
+      # NOTE: a build/test here fails until the JS SDK is published with any
+      # new surface this server uses (e.g. the `search` tool needs the SDK's
+      # `search()`), which is the correct gate — publish the SDK first.
       - run: npm ci
       - run: npm run typecheck
-      - run: npm run build
+      # `npm test` builds (rimraf + tsc) then runs the stdio/HTTP tool-parity
+      # test, so it covers the deployable dist AND the continuity guarantee.
+      - run: npm test
diff --git a/.gitignore b/.gitignore
index 397b0d9..fa75087 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@ dist/
 *.log
 .env
 .DS_Store
+.idea/
diff --git a/README.md b/README.md
index a94180b..05b7f0e 100644
--- a/README.md
+++ b/README.md
@@ -27,8 +27,35 @@ hints, and polymorphic job polling.
 
 ## Setup
 
-You need a WellMarked API key (`wm_...`) — generate one at
-[wellmarked.io](https://wellmarked.io).
+There are two ways to connect, depending on your host:
+
+- **Remote (hosted, OAuth)** — for Claude.ai and other hosts that support
+  remote MCP servers. No API key to copy; you sign in and authorize in the
+  browser. See [Remote server](#remote-server-hosted--oauth) below.
+- **Local (stdio, API key)** — for Claude Desktop, Claude Code, Cursor, and any
+  host that launches a local server. You supply a `wm_...` key via the
+  environment. Generate one at [wellmarked.io](https://wellmarked.io).
+
+### Remote server (hosted — OAuth)
+
+Add WellMarked as a **custom connector** and point it at:
+
+```
+https://mcp.wellmarked.io/mcp
+```
+
+Your host discovers the authorization server automatically (via
+`/.well-known/oauth-protected-resource`), walks you through signing in to
+WellMarked and approving the connection, and receives a scoped, expiring token —
+no key to paste. The connector can `extract`, `bulk`, and `crawl`; it cannot
+mint or rotate credentials.
+
+In **Claude.ai**: Settings → Connectors → Add custom connector → paste the URL
+above → follow the sign-in and consent prompts.
+
+The remote server speaks MCP over Streamable HTTP and, under the hood, calls the
+same tools as the local server — the tool surface is identical (enforced by the
+parity test in `test/`).
 
 ### Claude Desktop
 
@@ -61,20 +88,36 @@ Any host that launches MCP servers over stdio works — point it at
 
 ## Environment variables
 
+Local (stdio) server:
+
 | Variable | Required | Description |
 | --- | --- | --- |
 | `WELLMARKED_API_KEY` | yes | Your `wm_...` API key. |
 | `WELLMARKED_BASE_URL` | no | Override the API base URL (self-hosted instances). |
 | `WELLMARKED_TIMEOUT_MS` | no | Per-request timeout in ms (default `30000`). |
 
+Remote (Streamable HTTP) server — only if you self-host it (`npm run start:http`):
+
+| Variable | Required | Description |
+| --- | --- | --- |
+| `PORT` | no | Listen port (default `3000`; Railway sets it). |
+| `WELLMARKED_BASE_URL` | no | API base URL — also the OAuth authorization server. |
+| `MCP_PUBLIC_URL` | no | This server's public URL, used in the protected-resource metadata. Derived from the request Host if unset. |
+| `WELLMARKED_TIMEOUT_MS` | no | Per-request timeout in ms. |
+
+The remote server takes **no** `WELLMARKED_API_KEY` — each request authenticates
+with its own OAuth bearer token.
+
 ## Develop locally
 
 ```bash
 npm install
 npm run build        # compile TypeScript to dist/
 npm start            # run the compiled server on stdio
+npm run start:http   # OR run the remote (Streamable HTTP) server
+npm test             # tool-list parity across both entry points
 
-# Inspect it interactively with the MCP Inspector:
+# Inspect the stdio server interactively with the MCP Inspector:
 WELLMARKED_API_KEY=wm_... npx @modelcontextprotocol/inspector node dist/index.js
 ```
 
diff --git a/package.json b/package.json
index fb341f6..ff15c22 100644
--- a/package.json
+++ b/package.json
@@ -44,8 +44,10 @@
   "scripts": {
     "build": "rimraf dist && tsc -p tsconfig.json",
     "start": "node dist/index.js",
+    "start:http": "node dist/http.js",
     "dev": "tsc -p tsconfig.json --watch",
     "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "npm run build && node --test \"test/**/*.test.mjs\"",
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
diff --git a/railway.toml b/railway.toml
new file mode 100644
index 0000000..86fabcd
--- /dev/null
+++ b/railway.toml
@@ -0,0 +1,18 @@
+# The remote MCP server runs as its OWN Railway service (Phase 7) — separate
+# from the API and website. It's a stateless Node process; Nixpacks detects the
+# package.json, installs, and runs `npm run build`, then the startCommand below.
+#
+# The default `npm start` is the STDIO entry point (dist/index.js) for local
+# hosts — wrong for a hosted service — so the startCommand overrides it with the
+# Streamable HTTP entry point. Railway injects PORT; http.ts reads it.
+#
+# Required env: none. Optional: WELLMARKED_BASE_URL (defaults to the public API),
+# MCP_PUBLIC_URL (defaults to the request Host), WELLMARKED_TIMEOUT_MS.
+[build]
+builder = "NIXPACKS"
+
+[deploy]
+startCommand = "npm run start:http"
+healthcheckPath = "/health"
+healthcheckTimeout = 30
+restartPolicyType = "on_failure"
diff --git a/src/http.ts b/src/http.ts
new file mode 100644
index 0000000..09a7754
--- /dev/null
+++ b/src/http.ts
@@ -0,0 +1,220 @@
+#!/usr/bin/env node
+/**
+ * Remote entry point for the WellMarked MCP server — Streamable HTTP + OAuth
+ * (Phase 7).
+ *
+ * A sibling to index.ts (stdio). It speaks MCP over HTTP so hosted clients
+ * (Claude.ai custom connectors, etc.) can reach it without launching a local
+ * process, and it authenticates with OAuth 2.1 bearer tokens instead of an
+ * env-var API key.
+ *
+ * The seam that makes this small: it calls the SAME `createServer` as the
+ * stdio entry point, once per request, with the request's own bearer token as
+ * the API key. No tool code changes — the tool surface is transport-independent
+ * (see the parity test in test/).
+ *
+ * OAuth roles (RFC 9728 / the MCP authorization spec):
+ *   - This process is the RESOURCE SERVER. It publishes
+ *     `/.well-known/oauth-protected-resource` pointing at the WellMarked API as
+ *     the authorization server, and rejects unauthenticated/expired requests
+ *     with `401 WWW-Authenticate: Bearer resource_metadata=...` so the client
+ *     knows to run (or refresh) the OAuth flow.
+ *   - The WellMarked API is the AUTHORIZATION SERVER (api/routes/oauth.py).
+ *   - An access token is just a `wm_...` key with an expiry, so validation is a
+ *     single authenticated probe to the API — no bespoke token introspection.
+ *
+ * Runs STATELESS: a fresh transport + server per request. Access tokens expire
+ * hourly, so binding one to a long-lived session would break mid-session;
+ * per-request binding always uses the token the client just presented, and our
+ * tools are all request/response (no server-initiated streaming to preserve).
+ *
+ * Environment:
+ *   - WELLMARKED_BASE_URL   (optional) — API base URL. Default the public API.
+ *   - MCP_PUBLIC_URL        (optional) — this server's public URL, used in the
+ *                            protected-resource metadata. Derived from the
+ *                            request Host if unset.
+ *   - PORT                  (optional) — listen port. Default 3000 (Railway sets it).
+ *   - WELLMARKED_TIMEOUT_MS (optional) — per-request timeout in ms.
+ */
+import { createServer as createHttpServer, type IncomingMessage, type ServerResponse } from "node:http";
+
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import type { AuthInfo } from "@modelcontextprotocol/sdk/server/auth/types.js";
+
+import { createServer } from "./server.js";
+
+const API_BASE_URL = (process.env.WELLMARKED_BASE_URL || "https://api.wellmarked.io").replace(/\/+$/, "");
+const PORT = Number.parseInt(process.env.PORT || "3000", 10);
+
+// The scopes an OAuth connector token carries — the MCP tool surface. Mirrors
+// api/services/oauth.OAUTH_SCOPES. Advertised in the protected-resource
+// metadata; the API is the one that actually enforces them per tool.
+const OAUTH_SCOPES = ["extract", "bulk", "crawl"];
+
+function resolveTimeout(): number | undefined {
+  const raw = process.env.WELLMARKED_TIMEOUT_MS;
+  if (!raw) return undefined;
+  const n = Number.parseInt(raw, 10);
+  return Number.isFinite(n) && n > 0 ? n : undefined;
+}
+const TIMEOUT_MS = resolveTimeout();
+
+function publicBaseUrl(req: IncomingMessage): string {
+  if (process.env.MCP_PUBLIC_URL) return process.env.MCP_PUBLIC_URL.replace(/\/+$/, "");
+  const proto = String(req.headers["x-forwarded-proto"] || "https").split(",")[0].trim();
+  const host = req.headers.host || `localhost:${PORT}`;
+  return `${proto}://${host}`;
+}
+
+function resourceMetadataUrl(req: IncomingMessage): string {
+  return `${publicBaseUrl(req)}/.well-known/oauth-protected-resource`;
+}
+
+function setCors(res: ServerResponse): void {
+  // Tokens are bearer, not cookies, so a wildcard origin is safe (no
+  // credentialed CORS). Expose WWW-Authenticate so a browser client can read
+  // the resource_metadata hint that starts the OAuth flow.
+  res.setHeader("Access-Control-Allow-Origin", "*");
+  res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
+  res.setHeader("Access-Control-Allow-Headers", "Authorization, Content-Type, Mcp-Session-Id, Mcp-Protocol-Version");
+  res.setHeader("Access-Control-Expose-Headers", "WWW-Authenticate, Mcp-Session-Id");
+}
+
+function sendJson(res: ServerResponse, status: number, body: unknown): void {
+  const payload = JSON.stringify(body);
+  res.writeHead(status, { "Content-Type": "application/json" });
+  res.end(payload);
+}
+
+function unauthorized(req: IncomingMessage, res: ServerResponse, invalid: boolean): void {
+  // RFC 6750 / MCP: point the client at the protected-resource metadata so it
+  // can discover the authorization server and (re)authorize. `invalid_token`
+  // distinguishes an expired/revoked token (refresh) from a missing one.
+  const params = [`resource_metadata="${resourceMetadataUrl(req)}"`];
+  if (invalid) params.unshift(`error="invalid_token"`);
+  res.setHeader("WWW-Authenticate", `Bearer ${params.join(", ")}`);
+  sendJson(res, 401, {
+    error: invalid ? "invalid_token" : "missing_token",
+    error_description: invalid
+      ? "The access token is invalid or expired."
+      : "Authorization required. Provide a Bearer access token.",
+  });
+}
+
+/**
+ * Validate a bearer token against the WellMarked API. An access token is a
+ * `wm_...` key, so a single authenticated probe to an unmetered, no-scope
+ * endpoint (`/usage`) tells us validity: 200 → valid, 401/403 → invalid/expired,
+ * anything else (or a network failure) → the API is unavailable (fail closed).
+ */
+async function verifyToken(token: string): Promise<"valid" | "invalid" | "unavailable"> {
+  try {
+    const resp = await fetch(`${API_BASE_URL}/usage`, {
+      headers: { Authorization: `Bearer ${token}` },
+    });
+    if (resp.status === 200) return "valid";
+    if (resp.status === 401 || resp.status === 403) return "invalid";
+    return "unavailable";
+  } catch {
+    return "unavailable";
+  }
+}
+
+function readBody(req: IncomingMessage): Promise<unknown> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    req.on("data", (c: Buffer) => chunks.push(c));
+    req.on("end", () => {
+      const raw = Buffer.concat(chunks).toString("utf8");
+      if (!raw) return resolve(undefined);
+      try {
+        resolve(JSON.parse(raw));
+      } catch (e) {
+        reject(e);
+      }
+    });
+    req.on("error", reject);
+  });
+}
+
+async function handleMcp(req: IncomingMessage, res: ServerResponse): Promise<void> {
+  const header = req.headers.authorization || "";
+  const token = header.startsWith("Bearer ") ? header.slice("Bearer ".length).trim() : "";
+  if (!token) return unauthorized(req, res, false);
+
+  const verdict = await verifyToken(token);
+  if (verdict === "invalid") return unauthorized(req, res, true);
+  if (verdict === "unavailable") {
+    return sendJson(res, 503, {
+      error: "service_unavailable",
+      error_description: "Unable to reach the WellMarked API to validate the token.",
+    });
+  }
+
+  let body: unknown;
+  try {
+    body = await readBody(req);
+  } catch {
+    // JSON-RPC parse error.
+    return sendJson(res, 400, { jsonrpc: "2.0", error: { code: -32700, message: "Parse error" }, id: null });
+  }
+
+  // Stateless: one transport + server per request, keyed on this request's token.
+  const server = createServer({ apiKey: token, baseUrl: API_BASE_URL, timeoutMs: TIMEOUT_MS });
+  const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
+  res.on("close", () => {
+    void transport.close();
+    void server.close();
+  });
+  await server.connect(transport);
+
+  const auth: AuthInfo = { token, clientId: "wellmarked-mcp", scopes: OAUTH_SCOPES };
+  const authedReq = req as IncomingMessage & { auth?: AuthInfo };
+  authedReq.auth = auth;
+  await transport.handleRequest(authedReq, res, body);
+}
+
+const httpServer = createHttpServer((req, res) => {
+  setCors(res);
+  if (req.method === "OPTIONS") {
+    res.writeHead(204);
+    res.end();
+    return;
+  }
+
+  const url = new URL(req.url || "/", "http://localhost");
+
+  if (req.method === "GET" && url.pathname === "/health") {
+    return sendJson(res, 200, { status: "ok" });
+  }
+
+  if (req.method === "GET" && url.pathname === "/.well-known/oauth-protected-resource") {
+    return sendJson(res, 200, {
+      resource: `${publicBaseUrl(req)}/mcp`,
+      authorization_servers: [API_BASE_URL],
+      scopes_supported: OAUTH_SCOPES,
+      bearer_methods_supported: ["header"],
+    });
+  }
+
+  if (url.pathname === "/mcp") {
+    if (req.method === "POST") {
+      void handleMcp(req, res).catch((err) => {
+        process.stderr.write(`[wellmarked-mcp] request failed: ${err instanceof Error ? err.message : String(err)}\n`);
+        if (!res.headersSent) {
+          sendJson(res, 500, { jsonrpc: "2.0", error: { code: -32603, message: "Internal error" }, id: null });
+        }
+      });
+      return;
+    }
+    // Stateless mode has no standalone SSE stream or session teardown.
+    res.setHeader("Allow", "POST");
+    return sendJson(res, 405, { error: "method_not_allowed" });
+  }
+
+  sendJson(res, 404, { error: "not_found" });
+});
+
+httpServer.listen(PORT, () => {
+  process.stderr.write(`[wellmarked-mcp] Streamable HTTP listening on :${PORT} (API ${API_BASE_URL})\n`);
+});
diff --git a/src/server.ts b/src/server.ts
index cc50211..786aae3 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -31,6 +31,7 @@ import {
   type BulkJob,
   type CrawlJob,
   type ExtractResult,
+  type SearchResults,
   type Usage,
 } from "wellmarked";
 
@@ -80,6 +81,54 @@ function iso(d: Date | null): string {
   return d ? d.toISOString() : "—";
 }
 
+// ── Compliance overrides ────────────────────────────────────────────────────
+// Shared across extract / bulk / crawl. These can only NARROW the API key's own
+// policy server-side (add denies, restrict domains, upgrade robots to strict),
+// never widen it — so exposing them to an agent is safe.
+const policyInputSchema = {
+  allow_domains: z
+    .array(z.string())
+    .optional()
+    .describe(
+      "Restrict this request to these domains (and their subdomains). Can only " +
+        "narrow the key's own allow-list, never widen it.",
+    ),
+  deny_patterns: z
+    .array(z.string())
+    .optional()
+    .describe(
+      "Extra deny globs for this request, matched against the hostname and the " +
+        "full URL. Added to the key's deny list.",
+    ),
+  respect_robots: z
+    .enum(["strict", "lax"])
+    .optional()
+    .describe(
+      "'strict' honors robots.txt on this request (extract/bulk too, not just " +
+        "crawl); can tighten the key's setting but never loosen it.",
+    ),
+};
+
+interface PolicyToolArgs {
+  allow_domains?: string[];
+  deny_patterns?: string[];
+  respect_robots?: "strict" | "lax";
+}
+
+/** Map the snake_case tool args to the SDK's camelCase policy options. Unset
+ * fields stay undefined; the SDK omits them so the key's policy is untouched. */
+function toPolicyOptions(a: PolicyToolArgs): {
+  allowDomains?: string[];
+  denyPatterns?: string[];
+  respectRobots?: "strict" | "lax";
+} {
+  return {
+    allowDomains: a.allow_domains,
+    denyPatterns: a.deny_patterns,
+    respectRobots: a.respect_robots,
+  };
+}
+
 // ── Renderers ─────────────────────────────────────────────────────────────────
 // Render SDK objects as text an LLM can consume directly: the Markdown is the
 // payload the agent actually wants, so it goes in verbatim, with a compact
@@ -141,6 +190,25 @@ function renderJob(job: BulkJob | CrawlJob): string {
   return lines.join("\n");
 }
 
+function renderSearch(res: SearchResults): string {
+  const lines: string[] = [
+    `Search: ${res.query}  (${res.results.length} results)`,
+    `Request ID: ${res.requestId}`,
+  ];
+  for (const item of res.results) {
+    lines.push("");
+    const flag = item.ok ? "✓" : `✗ error: ${item.error ?? "unknown"}`;
+    lines.push(`## ${item.title || item.url}  ${flag}`);
+    lines.push(item.url);
+    if (item.snippet) lines.push(`> ${item.snippet}`);
+    if (item.ok && item.markdown) {
+      lines.push("");
+      lines.push(item.markdown);
+    }
+  }
+  return lines.join("\n");
+}
+
 function renderUsage(u: Usage): string {
   const pct = u.limit > 0 ? Math.round((u.used / u.limit) * 100) : 0;
   return [
@@ -186,6 +254,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
               "Needed for SPA / client-rendered pages. Requires a Pro+ plan " +
               "and the feature enabled on the API instance. Default false.",
           ),
+        ...policyInputSchema,
       },
       annotations: {
         readOnlyHint: true,
@@ -193,9 +262,12 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
         idempotentHint: true,
       },
     },
-    async ({ url, render_js }) => {
+    async ({ url, render_js, allow_domains, deny_patterns, respect_robots }) => {
       try {
-        const result = await client.extract(url, { renderJs: render_js });
+        const result = await client.extract(url, {
+          renderJs: render_js,
+          ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }),
+        });
         return ok(renderExtract(result));
       } catch (err) {
         return toErrorResult(err);
@@ -236,12 +308,16 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
           .positive()
           .optional()
           .describe("Max ms to wait when wait=true. Default 300000 (5 min)."),
+        ...policyInputSchema,
       },
       annotations: { openWorldHint: true },
     },
-    async ({ urls, render_js, wait, wait_timeout_ms }) => {
+    async ({ urls, render_js, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => {
       try {
-        let job = await client.bulk(urls, { renderJs: render_js });
+        let job = await client.bulk(urls, {
+          renderJs: render_js,
+          ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }),
+        });
         if (wait !== false) {
           job = (await client.waitForJob(job.jobId, {
             timeoutMs: wait_timeout_ms ?? 300_000,
@@ -291,14 +367,16 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
           .positive()
           .optional()
           .describe("Max ms to wait when wait=true. Default 300000 (5 min)."),
+        ...policyInputSchema,
       },
       annotations: { openWorldHint: true },
     },
-    async ({ url, depth, render_js, wait, wait_timeout_ms }) => {
+    async ({ url, depth, render_js, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => {
       try {
         let job: BulkJob | CrawlJob = await client.crawl(url, {
           depth,
           renderJs: render_js,
+          ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }),
         });
         if (wait === true) {
           job = await client.waitForJob(job.jobId, {
@@ -312,6 +390,50 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
     },
   );
 
+  // ── search ─────────────────────────────────────────────────────────────────
+  server.registerTool(
+    "search",
+    {
+      title: "Search the web and extract the results",
+      description:
+        "Search the web for a query and get each result page back as clean " +
+        "Markdown — search plus extraction in one synchronous call, no job to " +
+        "poll. Use this to answer a question from the live web when you don't " +
+        "already have the URLs. Returns up to num_results pages, each with a " +
+        "status (a slow or blocked page comes back as an error item, not a " +
+        "failure of the whole call). Requires a Pro+ plan.",
+      inputSchema: {
+        query: z.string().min(1).describe("The search query."),
+        num_results: z
+          .number()
+          .int()
+          .min(1)
+          .max(10)
+          .optional()
+          .describe("How many results to fetch + extract (1–10). Default 5."),
+        render_js: z
+          .boolean()
+          .optional()
+          .describe("Render JavaScript on each result page before extracting. Default false."),
+      },
+      annotations: {
+        readOnlyHint: true,
+        openWorldHint: true,
+      },
+    },
+    async ({ query, num_results, render_js }) => {
+      try {
+        const res = await client.search(query, {
+          numResults: num_results,
+          renderJs: render_js,
+        });
+        return ok(renderSearch(res));
+      } catch (err) {
+        return toErrorResult(err);
+      }
+    },
+  );
+
   // ── get_job ──────────────────────────────────────────────────────────────
   server.registerTool(
     "get_job",
diff --git a/test/tool-parity.test.mjs b/test/tool-parity.test.mjs
new file mode 100644
index 0000000..f3af393
--- /dev/null
+++ b/test/tool-parity.test.mjs
@@ -0,0 +1,50 @@
+// Tool-list parity across transports (Phase 7, step 4).
+//
+// The stdio entry point (src/index.ts) and the remote HTTP entry point
+// (src/http.ts) both build their McpServer from the SAME createServer(). The
+// whole point of that shared seam is that an agent sees the identical tools
+// whether it connects locally or over the network. This test pins that: it
+// lists the tools through a real client↔server transport pair and asserts the
+// exact surface. If a future edit forks tool definitions per transport, or
+// drops/renames a tool, this fails.
+//
+// Zero test deps on purpose (the repo had no runner): Node's built-in test
+// runner + the SDK's in-memory transport, against the built dist/. Run with
+// `npm test` (builds first).
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+
+import { createServer } from "../dist/server.js";
+
+const EXPECTED_TOOLS = ["bulk", "crawl", "extract", "get_job", "get_usage", "search", "wait_for_job"];
+
+// A well-formed dummy key: createServer builds a WellMarked client but makes no
+// network call, and listing tools is local (they're statically registered).
+const DUMMY_KEY = "wm_" + "0".repeat(40);
+
+async function toolNames() {
+  const server = createServer({ apiKey: DUMMY_KEY });
+  const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+  const client = new Client({ name: "parity-test", version: "0.0.0" });
+  await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]);
+  const { tools } = await client.listTools();
+  await client.close();
+  await server.close();
+  return tools.map((t) => t.name).sort();
+}
+
+test("the shared createServer exposes exactly the expected tool set", async () => {
+  const names = await toolNames();
+  assert.deepEqual(names, EXPECTED_TOOLS);
+});
+
+test("two independent createServer() instances (stdio vs HTTP) are identical", async () => {
+  // index.ts and http.ts each call createServer() independently. Comparing two
+  // fresh instances is a faithful stand-in for "stdio transport tools" vs
+  // "HTTP transport tools": same surface, transport-independent.
+  const [a, b] = await Promise.all([toolNames(), toolNames()]);
+  assert.deepEqual(a, b);
+});

From b9330ead29e40e139d4bde1408dbdda3784be4e2 Mon Sep 17 00:00:00 2001
From: Grady Dillon <gradydillon22@gmail.com>
Date: Mon, 20 Jul 2026 19:22:49 -0400
Subject: [PATCH 2/6] Add `format` to extract/bulk/crawl tools (Phase 4.5)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds a shared formatInputSchema (zod enum) to the three extraction tools.
`search` deliberately does not take one: it hands an agent N pages to
read, while the other formats serve pipelines that already know which URL
they want.

MCP tools return text, so renderContent() gives each format a legible
textual projection — an agent reading `[object Object]` is strictly worse
off than one reading markdown. Blocks and chunks keep their structure
visible (block type, token offsets), which is the reason a caller asked
for them. renderMetrics() adds a one-line tokens-saved summary when the
API reports it.

Build and the tool-parity test are clean.

NOTE: this repo's CI stays red for a reason unrelated to this change —
`wellmarked@^1.1.1` is unsatisfiable (npm has only 1.1.0) and the
lockfile still pins the SDK to file:../JS-TS-SDK. Locally that resolves
to the sibling checkout, which is why this builds here. See PR #13.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 src/server.ts | 76 +++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 70 insertions(+), 6 deletions(-)

diff --git a/src/server.ts b/src/server.ts
index 786aae3..db7b257 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -29,6 +29,7 @@ import {
   WellMarkedError,
   type WellMarkedOptions,
   type BulkJob,
+  type Content,
   type CrawlJob,
   type ExtractResult,
   type SearchResults,
@@ -109,6 +110,60 @@ const policyInputSchema = {
     ),
 };
 
+// ── Output format ───────────────────────────────────────────────────────────
+// Shared across extract / bulk / crawl. Kept off `search`, which always returns
+// markdown — search hands an agent N pages to read, and the other formats serve
+// pipelines that already know which URL they want.
+const formatInputSchema = {
+  format: z
+    .enum(["markdown", "json", "chunks", "html", "links"])
+    .optional()
+    .describe(
+      "Output format. 'markdown' (default) is clean prose; 'json' returns typed " +
+        "heading/paragraph/list/code blocks; 'chunks' returns contiguous " +
+        "500-token windows ready for embedding; 'html' returns the raw fetched " +
+        "HTML; 'links' returns every http(s) link on the page.",
+    ),
+};
+
+/**
+ * Render whichever content field a result carries into text.
+ *
+ * MCP tools return text, so each non-markdown format needs a legible textual
+ * projection — an agent reading `[object Object]` is strictly worse off than
+ * one reading markdown. Chunks and blocks keep their structure visible
+ * (offsets, block types) because that's the reason the caller asked for them.
+ */
+function renderContent(item: Content): string | null {
+  if (item.markdown !== null) return item.markdown;
+  if (item.html !== null) return item.html;
+  if (item.links !== null) return item.links.join("\n");
+  if (item.blocks !== null) {
+    return item.blocks
+      .map((b) => {
+        const label = b.type === "heading" ? `heading h${b.level ?? 1}` : b.type;
+        return `[${label}] ${b.text}`;
+      })
+      .join("\n\n");
+  }
+  if (item.chunks !== null) {
+    return item.chunks
+      .map((c) => `[tokens ${c.startToken}-${c.endToken}]\n${c.text}`)
+      .join("\n\n");
+  }
+  return null;
+}
+
+/** One line of ROI metrics, when the API reported them. */
+function renderMetrics(item: Content): string | null {
+  const m = item.metrics;
+  if (!m || m.tokensSaved === null) return null;
+  return (
+    `Tokens: ${m.outputTokens} out of ${m.inputTokens} raw ` +
+    `(saved ${m.tokensSaved}, −${m.reductionPct}%)`
+  );
+}
+
 interface PolicyToolArgs {
   allow_domains?: string[];
   deny_patterns?: string[];
@@ -146,7 +201,9 @@ function renderExtract(r: ExtractResult): string {
   ]
     .filter(Boolean)
     .join("\n");
-  return `${header}\n\n---\n\n${r.markdown}`;
+  const metrics = renderMetrics(r);
+  const full = metrics ? `${header}\n${metrics}` : header;
+  return `${full}\n\n---\n\n${renderContent(r) ?? "(no content)"}`;
 }
 
 function renderJob(job: BulkJob | CrawlJob): string {
@@ -182,9 +239,10 @@ function renderJob(job: BulkJob | CrawlJob): string {
       "depth" in item ? `  (depth ${(item as { depth: number }).depth})` : "";
     const flag = item.ok ? "✓" : `✗ error: ${item.error ?? "unknown"}`;
     lines.push(`## ${item.url}${depth}  ${flag}`);
-    if (item.ok && item.markdown) {
+    const body = item.ok ? renderContent(item) : null;
+    if (body) {
       lines.push("");
-      lines.push(item.markdown);
+      lines.push(body);
     }
   }
   return lines.join("\n");
@@ -255,6 +313,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
               "and the feature enabled on the API instance. Default false.",
           ),
         ...policyInputSchema,
+        ...formatInputSchema,
       },
       annotations: {
         readOnlyHint: true,
@@ -262,10 +321,11 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
         idempotentHint: true,
       },
     },
-    async ({ url, render_js, allow_domains, deny_patterns, respect_robots }) => {
+    async ({ url, render_js, format, allow_domains, deny_patterns, respect_robots }) => {
       try {
         const result = await client.extract(url, {
           renderJs: render_js,
+          format,
           ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }),
         });
         return ok(renderExtract(result));
@@ -309,13 +369,15 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
           .optional()
           .describe("Max ms to wait when wait=true. Default 300000 (5 min)."),
         ...policyInputSchema,
+        ...formatInputSchema,
       },
       annotations: { openWorldHint: true },
     },
-    async ({ urls, render_js, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => {
+    async ({ urls, render_js, format, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => {
       try {
         let job = await client.bulk(urls, {
           renderJs: render_js,
+          format,
           ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }),
         });
         if (wait !== false) {
@@ -368,14 +430,16 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
           .optional()
           .describe("Max ms to wait when wait=true. Default 300000 (5 min)."),
         ...policyInputSchema,
+        ...formatInputSchema,
       },
       annotations: { openWorldHint: true },
     },
-    async ({ url, depth, render_js, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => {
+    async ({ url, depth, render_js, format, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => {
       try {
         let job: BulkJob | CrawlJob = await client.crawl(url, {
           depth,
           renderJs: render_js,
+          format,
           ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }),
         });
         if (wait === true) {

From cad246cf15fc30d5819532cc778e89de439d8e95 Mon Sep 17 00:00:00 2001
From: Grady Dillon <gradydillon22@gmail.com>
Date: Wed, 22 Jul 2026 10:32:22 -0400
Subject: [PATCH 3/6] Bump to 1.1.0 for the remote OAuth transport
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The remote Streamable-HTTP + OAuth server is a new feature since 1.0.0, so
this warrants a minor release. version.ts (reported in the MCP initialize
handshake) is hand-synced to package.json — kept in sync here.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 package.json   | 2 +-
 src/version.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index ff15c22..6e38db7 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "wellmarked-mcp",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Official Model Context Protocol (MCP) server for the WellMarked API — let AI agents convert any URL to clean Markdown, bulk-extract, and crawl sites.",
   "keywords": [
     "wellmarked",
diff --git a/src/version.ts b/src/version.ts
index e0fde21..030e623 100644
--- a/src/version.ts
+++ b/src/version.ts
@@ -3,4 +3,4 @@
  * Keep SERVER_VERSION in sync with the `version` field in package.json.
  */
 export const SERVER_NAME = "wellmarked-mcp";
-export const SERVER_VERSION = "1.0.0";
+export const SERVER_VERSION = "1.1.0";

From 2404fd9997bcf24bc0b60924fd7f66b5f9554266 Mon Sep 17 00:00:00 2001
From: Grady Dillon <gradydillon22@gmail.com>
Date: Wed, 22 Jul 2026 12:39:14 -0400
Subject: [PATCH 4/6] =?UTF-8?q?Remove=20WELLMARKED=5FBASE=5FURL=20?=
 =?UTF-8?q?=E2=80=94=20the=20server=20always=20targets=20the=20public=20AP?=
 =?UTF-8?q?I?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Product decision: no capability to point WellMarked tooling anywhere but
api.wellmarked.io. Both the stdio entrypoint and the remote Streamable-HTTP
server now hardcode the API base; the env var is gone from the docs. Also
tracks the SDK's removal of its baseUrl option (createServer forwards
options straight into new WellMarked()).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 README.md    | 2 --
 src/http.ts  | 5 ++---
 src/index.ts | 2 --
 3 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 05b7f0e..f4f9828 100644
--- a/README.md
+++ b/README.md
@@ -93,7 +93,6 @@ Local (stdio) server:
 | Variable | Required | Description |
 | --- | --- | --- |
 | `WELLMARKED_API_KEY` | yes | Your `wm_...` API key. |
-| `WELLMARKED_BASE_URL` | no | Override the API base URL (self-hosted instances). |
 | `WELLMARKED_TIMEOUT_MS` | no | Per-request timeout in ms (default `30000`). |
 
 Remote (Streamable HTTP) server — only if you self-host it (`npm run start:http`):
@@ -101,7 +100,6 @@ Remote (Streamable HTTP) server — only if you self-host it (`npm run start:htt
 | Variable | Required | Description |
 | --- | --- | --- |
 | `PORT` | no | Listen port (default `3000`; Railway sets it). |
-| `WELLMARKED_BASE_URL` | no | API base URL — also the OAuth authorization server. |
 | `MCP_PUBLIC_URL` | no | This server's public URL, used in the protected-resource metadata. Derived from the request Host if unset. |
 | `WELLMARKED_TIMEOUT_MS` | no | Per-request timeout in ms. |
 
diff --git a/src/http.ts b/src/http.ts
index 09a7754..6e1fa12 100644
--- a/src/http.ts
+++ b/src/http.ts
@@ -29,7 +29,6 @@
  * tools are all request/response (no server-initiated streaming to preserve).
  *
  * Environment:
- *   - WELLMARKED_BASE_URL   (optional) — API base URL. Default the public API.
  *   - MCP_PUBLIC_URL        (optional) — this server's public URL, used in the
  *                            protected-resource metadata. Derived from the
  *                            request Host if unset.
@@ -43,7 +42,7 @@ import type { AuthInfo } from "@modelcontextprotocol/sdk/server/auth/types.js";
 
 import { createServer } from "./server.js";
 
-const API_BASE_URL = (process.env.WELLMARKED_BASE_URL || "https://api.wellmarked.io").replace(/\/+$/, "");
+const API_BASE_URL = "https://api.wellmarked.io";
 const PORT = Number.parseInt(process.env.PORT || "3000", 10);
 
 // The scopes an OAuth connector token carries — the MCP tool surface. Mirrors
@@ -160,7 +159,7 @@ async function handleMcp(req: IncomingMessage, res: ServerResponse): Promise<voi
   }
 
   // Stateless: one transport + server per request, keyed on this request's token.
-  const server = createServer({ apiKey: token, baseUrl: API_BASE_URL, timeoutMs: TIMEOUT_MS });
+  const server = createServer({ apiKey: token, timeoutMs: TIMEOUT_MS });
   const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
   res.on("close", () => {
     void transport.close();
diff --git a/src/index.ts b/src/index.ts
index f35d121..cf83064 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -12,7 +12,6 @@
  *
  * Environment:
  *   - WELLMARKED_API_KEY   (required) — your `wm_...` key.
- *   - WELLMARKED_BASE_URL  (optional) — override the API base URL.
  *   - WELLMARKED_TIMEOUT_MS(optional) — per-request timeout in ms.
  */
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
@@ -31,7 +30,6 @@ async function main(): Promise<void> {
   try {
     server = createServer({
       apiKey: process.env.WELLMARKED_API_KEY,
-      baseUrl: process.env.WELLMARKED_BASE_URL,
       timeoutMs: resolveTimeout(),
     });
   } catch (err) {

From 8d72801e7c4f860a0c06d83cf68cc48f508933b6 Mon Sep 17 00:00:00 2001
From: Grady Dillon <gradydillon22@gmail.com>
Date: Wed, 22 Jul 2026 14:49:55 -0400
Subject: [PATCH 5/6] search tool carries format + policy overrides; results
 render any format

Mirrors the API: search takes the same extraction parameter set as extract/
bulk/crawl. renderSearch now goes through the shared renderContent path so
chunks/json/html/links results project to legible text instead of only
markdown. Format description notes the Pro+ gate on json/chunks.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 src/server.ts | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/server.ts b/src/server.ts
index db7b257..b0acaa3 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -122,7 +122,8 @@ const formatInputSchema = {
       "Output format. 'markdown' (default) is clean prose; 'json' returns typed " +
         "heading/paragraph/list/code blocks; 'chunks' returns contiguous " +
         "500-token windows ready for embedding; 'html' returns the raw fetched " +
-        "HTML; 'links' returns every http(s) link on the page.",
+        "HTML; 'links' returns every http(s) link on the page. 'json' and " +
+        "'chunks' require a Pro, Growth, or Enterprise plan.",
     ),
 };
 
@@ -259,9 +260,14 @@ function renderSearch(res: SearchResults): string {
     lines.push(`## ${item.title || item.url}  ${flag}`);
     lines.push(item.url);
     if (item.snippet) lines.push(`> ${item.snippet}`);
-    if (item.ok && item.markdown) {
-      lines.push("");
-      lines.push(item.markdown);
+    if (item.ok) {
+      // Format-agnostic: search carries the same format param as extract, so
+      // a chunks/json/html/links result renders through the shared path.
+      const content = renderContent(item);
+      if (content) {
+        lines.push("");
+        lines.push(content);
+      }
     }
   }
   return lines.join("\n");
@@ -479,17 +485,21 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
           .boolean()
           .optional()
           .describe("Render JavaScript on each result page before extracting. Default false."),
+        ...policyInputSchema,
+        ...formatInputSchema,
       },
       annotations: {
         readOnlyHint: true,
         openWorldHint: true,
       },
     },
-    async ({ query, num_results, render_js }) => {
+    async ({ query, num_results, render_js, format, allow_domains, deny_patterns, respect_robots }) => {
       try {
         const res = await client.search(query, {
           numResults: num_results,
           renderJs: render_js,
+          format,
+          ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }),
         });
         return ok(renderSearch(res));
       } catch (err) {

From 91bf57b6b8d21c645adfe19609e4988abdd30278 Mon Sep 17 00:00:00 2001
From: Grady Dillon <gradydillon22@gmail.com>
Date: Wed, 22 Jul 2026 15:57:16 -0400
Subject: [PATCH 6/6] Expose retry on extract/bulk/crawl and max_pages on crawl
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

retry: server-side re-attempts on target_timeout (fresh connection per
attempt, default 0). Shared schema across the three extraction tools;
deliberately kept off search — the API's 15s per-result deadline can't
absorb a retry, so the parameter doesn't exist there.

max_pages narrows the plan's crawl page cap (never widens it).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 src/server.ts | 39 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/src/server.ts b/src/server.ts
index b0acaa3..801edaa 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -127,6 +127,23 @@ const formatInputSchema = {
     ),
 };
 
+// ── Timeout retries ─────────────────────────────────────────────────────────
+// Shared across extract / bulk / crawl. Kept off `search`: its 15s per-result
+// deadline can't absorb a retry, so the API doesn't take one there.
+const retryInputSchema = {
+  retry: z
+    .number()
+    .int()
+    .min(0)
+    .optional()
+    .describe(
+      "Server-side re-attempts when the target times out, each on a fresh " +
+        "connection. Default 0 (one attempt). On the synchronous extract " +
+        "each timed-out attempt takes 20-30s, so prefer bulk for aggressive " +
+        "values.",
+    ),
+};
+
 /**
  * Render whichever content field a result carries into text.
  *
@@ -320,6 +337,7 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
           ),
         ...policyInputSchema,
         ...formatInputSchema,
+        ...retryInputSchema,
       },
       annotations: {
         readOnlyHint: true,
@@ -327,11 +345,12 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
         idempotentHint: true,
       },
     },
-    async ({ url, render_js, format, allow_domains, deny_patterns, respect_robots }) => {
+    async ({ url, render_js, format, retry, allow_domains, deny_patterns, respect_robots }) => {
       try {
         const result = await client.extract(url, {
           renderJs: render_js,
           format,
+          retry,
           ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }),
         });
         return ok(renderExtract(result));
@@ -376,14 +395,16 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
           .describe("Max ms to wait when wait=true. Default 300000 (5 min)."),
         ...policyInputSchema,
         ...formatInputSchema,
+        ...retryInputSchema,
       },
       annotations: { openWorldHint: true },
     },
-    async ({ urls, render_js, format, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => {
+    async ({ urls, render_js, format, retry, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => {
       try {
         let job = await client.bulk(urls, {
           renderJs: render_js,
           format,
+          retry,
           ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }),
         });
         if (wait !== false) {
@@ -435,17 +456,29 @@ export function createServer(options: WellMarkedOptions = {}): McpServer {
           .positive()
           .optional()
           .describe("Max ms to wait when wait=true. Default 300000 (5 min)."),
+        max_pages: z
+          .number()
+          .int()
+          .min(1)
+          .optional()
+          .describe(
+            "Stop the crawl after this many successful pages. Can only " +
+              "narrow the plan's page cap, never widen it.",
+          ),
         ...policyInputSchema,
         ...formatInputSchema,
+        ...retryInputSchema,
       },
       annotations: { openWorldHint: true },
     },
-    async ({ url, depth, render_js, format, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => {
+    async ({ url, depth, render_js, format, retry, max_pages, wait, wait_timeout_ms, allow_domains, deny_patterns, respect_robots }) => {
       try {
         let job: BulkJob | CrawlJob = await client.crawl(url, {
           depth,
           renderJs: render_js,
           format,
+          retry,
+          maxPages: max_pages,
           ...toPolicyOptions({ allow_domains, deny_patterns, respect_robots }),
         });
         if (wait === true) {