Add remote (Streamable HTTP + OAuth) server, search tool, and tests

.github/workflows/ci.yml

@@ -24,6 +24,11 @@ jobs:
           cache: npm
       # Resolves "wellmarked" from npm (the published JS/TS SDK), not a
       # local file: path — so this builds in an isolated repo checkout.
+      # NOTE: a build/test here fails until the JS SDK is published with any
+      # new surface this server uses (e.g. the `search` tool needs the SDK's
+      # `search()`), which is the correct gate — publish the SDK first.
       - run: npm ci
       - run: npm run typecheck
-      - run: npm run build
+      # `npm test` builds (rimraf + tsc) then runs the stdio/HTTP tool-parity
+      # test, so it covers the deployable dist AND the continuity guarantee.
+      - run: npm test

.gitignore

@@ -3,3 +3,4 @@ dist/
 *.log
 .env
 .DS_Store
+.idea/

README.md

@@ -27,8 +27,35 @@ hints, and polymorphic job polling.
 
 ## Setup
 
-You need a WellMarked API key (`wm_...`) — generate one at
-[wellmarked.io](https://wellmarked.io).
+There are two ways to connect, depending on your host:
+
+- **Remote (hosted, OAuth)** — for Claude.ai and other hosts that support
+  remote MCP servers. No API key to copy; you sign in and authorize in the
+  browser. See [Remote server](#remote-server-hosted--oauth) below.
+- **Local (stdio, API key)** — for Claude Desktop, Claude Code, Cursor, and any
+  host that launches a local server. You supply a `wm_...` key via the
+  environment. Generate one at [wellmarked.io](https://wellmarked.io).
+
+### Remote server (hosted — OAuth)
+
+Add WellMarked as a **custom connector** and point it at:
+
+```
+https://mcp.wellmarked.io/mcp
+```
+
+Your host discovers the authorization server automatically (via
+`/.well-known/oauth-protected-resource`), walks you through signing in to
+WellMarked and approving the connection, and receives a scoped, expiring token —
+no key to paste. The connector can `extract`, `bulk`, and `crawl`; it cannot
+mint or rotate credentials.
+
+In **Claude.ai**: Settings → Connectors → Add custom connector → paste the URL
+above → follow the sign-in and consent prompts.
+
+The remote server speaks MCP over Streamable HTTP and, under the hood, calls the
+same tools as the local server — the tool surface is identical (enforced by the
+parity test in `test/`).
 
 ### Claude Desktop
 
@@ -61,20 +88,34 @@ Any host that launches MCP servers over stdio works — point it at
 
 ## Environment variables
 
+Local (stdio) server:
+
 | Variable | Required | Description |
 | --- | --- | --- |
 | `WELLMARKED_API_KEY` | yes | Your `wm_...` API key. |
-| `WELLMARKED_BASE_URL` | no | Override the API base URL (self-hosted instances). |
 | `WELLMARKED_TIMEOUT_MS` | no | Per-request timeout in ms (default `30000`). |
 
+Remote (Streamable HTTP) server — only if you self-host it (`npm run start:http`):
+
+| Variable | Required | Description |
+| --- | --- | --- |
+| `PORT` | no | Listen port (default `3000`; Railway sets it). |
+| `MCP_PUBLIC_URL` | no | This server's public URL, used in the protected-resource metadata. Derived from the request Host if unset. |
+| `WELLMARKED_TIMEOUT_MS` | no | Per-request timeout in ms. |
+
+The remote server takes **no** `WELLMARKED_API_KEY` — each request authenticates
+with its own OAuth bearer token.
+
 ## Develop locally
 
 ```bash
 npm install
 npm run build        # compile TypeScript to dist/
 npm start            # run the compiled server on stdio
+npm run start:http   # OR run the remote (Streamable HTTP) server
+npm test             # tool-list parity across both entry points
 
-# Inspect it interactively with the MCP Inspector:
+# Inspect the stdio server interactively with the MCP Inspector:
 WELLMARKED_API_KEY=wm_... npx @modelcontextprotocol/inspector node dist/index.js
 ```
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "wellmarked-mcp",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Official Model Context Protocol (MCP) server for the WellMarked API — let AI agents convert any URL to clean Markdown, bulk-extract, and crawl sites.",
   "keywords": [
     "wellmarked",
@@ -44,8 +44,10 @@
   "scripts": {
     "build": "rimraf dist && tsc -p tsconfig.json",
     "start": "node dist/index.js",
+    "start:http": "node dist/http.js",
     "dev": "tsc -p tsconfig.json --watch",
     "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "npm run build && node --test \"test/**/*.test.mjs\"",
     "prepublishOnly": "npm run build"
   },
   "dependencies": {

railway.toml

@@ -0,0 +1,18 @@
+# The remote MCP server runs as its OWN Railway service (Phase 7) — separate
+# from the API and website. It's a stateless Node process; Nixpacks detects the
+# package.json, installs, and runs `npm run build`, then the startCommand below.
+#
+# The default `npm start` is the STDIO entry point (dist/index.js) for local
+# hosts — wrong for a hosted service — so the startCommand overrides it with the
+# Streamable HTTP entry point. Railway injects PORT; http.ts reads it.
+#
+# Required env: none. Optional: WELLMARKED_BASE_URL (defaults to the public API),
+# MCP_PUBLIC_URL (defaults to the request Host), WELLMARKED_TIMEOUT_MS.
+[build]
+builder = "NIXPACKS"
+
+[deploy]
+startCommand = "npm run start:http"
+healthcheckPath = "/health"
+healthcheckTimeout = 30
+restartPolicyType = "on_failure"

src/http.ts

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+/**
+ * Remote entry point for the WellMarked MCP server — Streamable HTTP + OAuth
+ * (Phase 7).
+ *
+ * A sibling to index.ts (stdio). It speaks MCP over HTTP so hosted clients
+ * (Claude.ai custom connectors, etc.) can reach it without launching a local
+ * process, and it authenticates with OAuth 2.1 bearer tokens instead of an
+ * env-var API key.
+ *
+ * The seam that makes this small: it calls the SAME `createServer` as the
+ * stdio entry point, once per request, with the request's own bearer token as
+ * the API key. No tool code changes — the tool surface is transport-independent
+ * (see the parity test in test/).
+ *
+ * OAuth roles (RFC 9728 / the MCP authorization spec):
+ *   - This process is the RESOURCE SERVER. It publishes
+ *     `/.well-known/oauth-protected-resource` pointing at the WellMarked API as
+ *     the authorization server, and rejects unauthenticated/expired requests
+ *     with `401 WWW-Authenticate: Bearer resource_metadata=...` so the client
+ *     knows to run (or refresh) the OAuth flow.
+ *   - The WellMarked API is the AUTHORIZATION SERVER (api/routes/oauth.py).
+ *   - An access token is just a `wm_...` key with an expiry, so validation is a
+ *     single authenticated probe to the API — no bespoke token introspection.
+ *
+ * Runs STATELESS: a fresh transport + server per request. Access tokens expire
+ * hourly, so binding one to a long-lived session would break mid-session;
+ * per-request binding always uses the token the client just presented, and our
+ * tools are all request/response (no server-initiated streaming to preserve).
+ *
+ * Environment:
+ *   - MCP_PUBLIC_URL        (optional) — this server's public URL, used in the
+ *                            protected-resource metadata. Derived from the
+ *                            request Host if unset.
+ *   - PORT                  (optional) — listen port. Default 3000 (Railway sets it).
+ *   - WELLMARKED_TIMEOUT_MS (optional) — per-request timeout in ms.
+ */
+import { createServer as createHttpServer, type IncomingMessage, type ServerResponse } from "node:http";
+
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import type { AuthInfo } from "@modelcontextprotocol/sdk/server/auth/types.js";
+
+import { createServer } from "./server.js";
+
+const API_BASE_URL = "https://api.wellmarked.io";
+const PORT = Number.parseInt(process.env.PORT || "3000", 10);
+
+// The scopes an OAuth connector token carries — the MCP tool surface. Mirrors
+// api/services/oauth.OAUTH_SCOPES. Advertised in the protected-resource
+// metadata; the API is the one that actually enforces them per tool.
+const OAUTH_SCOPES = ["extract", "bulk", "crawl"];
+
+function resolveTimeout(): number | undefined {
+  const raw = process.env.WELLMARKED_TIMEOUT_MS;
+  if (!raw) return undefined;
+  const n = Number.parseInt(raw, 10);
+  return Number.isFinite(n) && n > 0 ? n : undefined;
+}
+const TIMEOUT_MS = resolveTimeout();
+
+function publicBaseUrl(req: IncomingMessage): string {
+  if (process.env.MCP_PUBLIC_URL) return process.env.MCP_PUBLIC_URL.replace(/\/+$/, "");
+  const proto = String(req.headers["x-forwarded-proto"] || "https").split(",")[0].trim();
+  const host = req.headers.host || `localhost:${PORT}`;
+  return `${proto}://${host}`;
+}
+
+function resourceMetadataUrl(req: IncomingMessage): string {
+  return `${publicBaseUrl(req)}/.well-known/oauth-protected-resource`;
+}
+
+function setCors(res: ServerResponse): void {
+  // Tokens are bearer, not cookies, so a wildcard origin is safe (no
+  // credentialed CORS). Expose WWW-Authenticate so a browser client can read
+  // the resource_metadata hint that starts the OAuth flow.
+  res.setHeader("Access-Control-Allow-Origin", "*");
+  res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
+  res.setHeader("Access-Control-Allow-Headers", "Authorization, Content-Type, Mcp-Session-Id, Mcp-Protocol-Version");
+  res.setHeader("Access-Control-Expose-Headers", "WWW-Authenticate, Mcp-Session-Id");
+}
+
+function sendJson(res: ServerResponse, status: number, body: unknown): void {
+  const payload = JSON.stringify(body);
+  res.writeHead(status, { "Content-Type": "application/json" });
+  res.end(payload);
+}
+
+function unauthorized(req: IncomingMessage, res: ServerResponse, invalid: boolean): void {
+  // RFC 6750 / MCP: point the client at the protected-resource metadata so it
+  // can discover the authorization server and (re)authorize. `invalid_token`
+  // distinguishes an expired/revoked token (refresh) from a missing one.
+  const params = [`resource_metadata="${resourceMetadataUrl(req)}"`];
+  if (invalid) params.unshift(`error="invalid_token"`);
+  res.setHeader("WWW-Authenticate", `Bearer ${params.join(", ")}`);
+  sendJson(res, 401, {
+    error: invalid ? "invalid_token" : "missing_token",
+    error_description: invalid
+      ? "The access token is invalid or expired."
+      : "Authorization required. Provide a Bearer access token.",
+  });
+}
+
+/**
+ * Validate a bearer token against the WellMarked API. An access token is a
+ * `wm_...` key, so a single authenticated probe to an unmetered, no-scope
+ * endpoint (`/usage`) tells us validity: 200 → valid, 401/403 → invalid/expired,
+ * anything else (or a network failure) → the API is unavailable (fail closed).
+ */
+async function verifyToken(token: string): Promise<"valid" | "invalid" | "unavailable"> {
+  try {
+    const resp = await fetch(`${API_BASE_URL}/usage`, {
+      headers: { Authorization: `Bearer ${token}` },
+    });
+    if (resp.status === 200) return "valid";
+    if (resp.status === 401 || resp.status === 403) return "invalid";
+    return "unavailable";
+  } catch {
+    return "unavailable";
+  }
+}
+
+function readBody(req: IncomingMessage): Promise<unknown> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    req.on("data", (c: Buffer) => chunks.push(c));
+    req.on("end", () => {
+      const raw = Buffer.concat(chunks).toString("utf8");
+      if (!raw) return resolve(undefined);
+      try {
+        resolve(JSON.parse(raw));
+      } catch (e) {
+        reject(e);
+      }
+    });
+    req.on("error", reject);
+  });
+}
+
+async function handleMcp(req: IncomingMessage, res: ServerResponse): Promise<void> {
+  const header = req.headers.authorization || "";
+  const token = header.startsWith("Bearer ") ? header.slice("Bearer ".length).trim() : "";
+  if (!token) return unauthorized(req, res, false);
+
+  const verdict = await verifyToken(token);
+  if (verdict === "invalid") return unauthorized(req, res, true);
+  if (verdict === "unavailable") {
+    return sendJson(res, 503, {
+      error: "service_unavailable",
+      error_description: "Unable to reach the WellMarked API to validate the token.",
+    });
+  }
+
+  let body: unknown;
+  try {
+    body = await readBody(req);
+  } catch {
+    // JSON-RPC parse error.
+    return sendJson(res, 400, { jsonrpc: "2.0", error: { code: -32700, message: "Parse error" }, id: null });
+  }
+
+  // Stateless: one transport + server per request, keyed on this request's token.
+  const server = createServer({ apiKey: token, timeoutMs: TIMEOUT_MS });
+  const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
+  res.on("close", () => {
+    void transport.close();
+    void server.close();
+  });
+  await server.connect(transport);
+
+  const auth: AuthInfo = { token, clientId: "wellmarked-mcp", scopes: OAUTH_SCOPES };
+  const authedReq = req as IncomingMessage & { auth?: AuthInfo };
+  authedReq.auth = auth;
+  await transport.handleRequest(authedReq, res, body);
+}
+
+const httpServer = createHttpServer((req, res) => {
+  setCors(res);
+  if (req.method === "OPTIONS") {
+    res.writeHead(204);
+    res.end();
+    return;
+  }
+
+  const url = new URL(req.url || "/", "http://localhost");
+
+  if (req.method === "GET" && url.pathname === "/health") {
+    return sendJson(res, 200, { status: "ok" });
+  }
+
+  if (req.method === "GET" && url.pathname === "/.well-known/oauth-protected-resource") {
+    return sendJson(res, 200, {
+      resource: `${publicBaseUrl(req)}/mcp`,
+      authorization_servers: [API_BASE_URL],
+      scopes_supported: OAUTH_SCOPES,
+      bearer_methods_supported: ["header"],
+    });
+  }
+
+  if (url.pathname === "/mcp") {
+    if (req.method === "POST") {
+      void handleMcp(req, res).catch((err) => {
+        process.stderr.write(`[wellmarked-mcp] request failed: ${err instanceof Error ? err.message : String(err)}\n`);
+        if (!res.headersSent) {
+          sendJson(res, 500, { jsonrpc: "2.0", error: { code: -32603, message: "Internal error" }, id: null });
+        }
+      });
+      return;
+    }
+    // Stateless mode has no standalone SSE stream or session teardown.
+    res.setHeader("Allow", "POST");
+    return sendJson(res, 405, { error: "method_not_allowed" });
+  }
+
+  sendJson(res, 404, { error: "not_found" });
+});
+
+httpServer.listen(PORT, () => {
+  process.stderr.write(`[wellmarked-mcp] Streamable HTTP listening on :${PORT} (API ${API_BASE_URL})\n`);
+});

src/index.ts

@@ -12,7 +12,6 @@
  *
  * Environment:
  *   - WELLMARKED_API_KEY   (required) — your `wm_...` key.
- *   - WELLMARKED_BASE_URL  (optional) — override the API base URL.
  *   - WELLMARKED_TIMEOUT_MS(optional) — per-request timeout in ms.
  */
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
@@ -31,7 +30,6 @@ async function main(): Promise<void> {
   try {
     server = createServer({
       apiKey: process.env.WELLMARKED_API_KEY,
-      baseUrl: process.env.WELLMARKED_BASE_URL,
       timeoutMs: resolveTimeout(),
     });
   } catch (err) {