diff --git a/README.md b/README.md index 81391dc..ea643cc 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,37 @@ Two things make the mutation run trustworthy here, both learned the hard way on - **A hanging mutant must fail, not stall.** Every HTTP assertion carries an `AbortSignal.timeout`, so a mutant that makes a request go unanswered is reported as killed rather than freezing the run. An earlier mutation run on this code had to be killed on a timeout instead of producing a number. - **Do not quote a literal in a comment next to the code it belongs to.** Stryker mutates string literals wherever they appear, comments included, which silently converts "mutant survived" into "mutant was never applied". +## Live e2e + +Every gate above is in-process: `createServer` over an in-memory transport, or the real express app on loopback with `globalThis.fetch` replaced. They prove this checkout behaves. They cannot see the build that is actually serving `mcp.ankr.com`, which is the gap where "green branch, stale pod" lives. + +`pnpm test:e2e` closes it. It runs `test/e2e/*.e2e.ts` against a **deployed** target and is deliberately outside the push gate and outside CI: it needs a credential, it costs real requests, and a failure in its parity group means "deploy this", not "fix this code". + +```sh +ANKR_RPC_KEY= pnpm test:e2e # against mcp.ankr.com +E2E_BASE_URL=http://127.0.0.1:3111 E2E_MGMT=0 pnpm test:e2e # against a local data plane +E2E_EXPECT_COMMIT= ANKR_RPC_KEY= pnpm test:e2e # pin the sha a release should be serving +``` + +| Var | Default | Purpose | +| ------------------- | ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `ANKR_RPC_KEY` | — | required; the suite fails rather than skipping without it | +| `E2E_BASE_URL` | `https://mcp.ankr.com` | target origin | +| `E2E_DATA_PATH` | `/rpc` | data-plane path | +| `E2E_MGMT_PATH` | `/mcp` | management-plane path | +| `E2E_MGMT` | (on) | `0` skips the management group, for a target that serves only the data plane | +| `E2E_EXPECT_COMMIT` | (unset) | the 40-hex sha the deployment should be serving. Unset, the suite still requires a commit suffix to be present, it just does not pin its value | + +Start the local target with `BUILD_COMMIT=$(git rev-parse HEAD)` if you want the whole suite green against it. Build identity is a property of the IMAGE BUILD, so a server started by hand without that variable answers a bare `0.2.0` and fails the identity test on purpose: that is the same signal a deployment built without `--build-arg BUILD_COMMIT` would give, and it is the regression the test exists to catch. + +Three groups, answering different questions: + +- **`live-data-plane.e2e.ts` — invariants.** Contracts any healthy deployment honours, old build or new: health, the 401s, session binding (a leaked `Mcp-Session-Id` is not authority), argument strictness _called_ rather than read off the schema, and one real `eth_blockNumber` proving the pod reaches a chain instead of only answering from its own process. Red here is an outage or a regression. +- **`live-parity.e2e.ts` — is the deployed build this commit?** The expectation is GENERATED from `src/`, not transcribed: `createServer` over an in-memory transport supplies the tool set, descriptions, schemas, annotations and instructions, and `createHttpApp` on loopback supplies the error wording. Red here means the deployment is behind — the code is fine. +- **`guard.e2e.ts` — the harness's own safety property.** The suite points at production, so its read-only limit is enforced in code (an allowlist of JSON-RPC methods, of tool names, and of `rpcCall` methods) rather than promised in a comment. These tests send nothing; they assert the guard refuses, and that it does not refuse everything. + +Run it against a local server built from the branch as well as against production. Parity passing locally and failing remotely is what tells you the difference is deployment, not code. + ## License MIT. diff --git a/package.json b/package.json index 0cbb44d..774c144 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "typecheck": "tsc --noEmit && tsc -p tsconfig.test.json", "check": "tsc --noEmit && eslint .", "test": "tsx --test test/*.test.ts", + "test:e2e": "tsx --test test/e2e/*.e2e.ts", "test:coverage": "COVERAGE_RUN=1 tsx --test --experimental-test-coverage --test-coverage-exclude='test/**' --test-coverage-lines=90 --test-coverage-branches=80 --test-coverage-functions=85 test/*.test.ts", "test:coverage:mgmt": "tsx --test --experimental-test-coverage --test-coverage-include='src/mgmt/**' --test-coverage-include='src/mgmt-http.ts' --test-coverage-include='src/deployMode.ts' --test-coverage-include='src/sessionRegistry.ts' --test-coverage-include='src/bodyLimit.ts' --test-coverage-lines=80 --test-coverage-branches=75 --test-coverage-functions=80 test/*.test.ts", "mutation": "stryker run", diff --git a/test/e2e/guard.e2e.ts b/test/e2e/guard.e2e.ts new file mode 100644 index 0000000..0539b30 --- /dev/null +++ b/test/e2e/guard.e2e.ts @@ -0,0 +1,119 @@ +// The read-only guard in test/e2e/liveTarget.ts, tested. +// +// This suite points at production, so "it only reads" is a safety property, and a +// safety property nothing checks is a comment. These tests send no request: they +// drive `send` with bodies it must refuse and assert that it refuses before any +// fetch happens. They live in the e2e glob because they are about the e2e +// harness, and they need no target beyond the configuration every file here +// already requires. +// +// The failure mode being prevented is concrete: someone adds a management or +// write call to this suite because it is "just one check", and it runs against +// real customer state on the first CI invocation. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { resolveTarget, send } from "./liveTarget.js"; + +const target = resolveTarget(); + +// If the guard ever lets one of these through, the request must still not leave +// the process — so fetch is replaced for the duration and its use is a failure +// in itself, rather than a live call with a comment saying it should not happen. +const withNoNetwork = async (fn: () => Promise): Promise => { + const original = globalThis.fetch; + let calls = 0; + globalThis.fetch = (() => { + calls += 1; + return Promise.reject(new Error("network reached")); + }) as typeof fetch; + try { + await fn(); + } finally { + globalThis.fetch = original; + } + assert.equal(calls, 0, "a refused request must never reach the network"); +}; + +test("a JSON-RPC method outside the read-only set is refused", async () => { + await withNoNetwork(async () => { + await assert.rejects( + send(target, { jsonrpc: "2.0", id: 1, method: "resources/subscribe" }), + /refusing to send "resources\/subscribe"/ + ); + }); +}); + +test("a tool outside the read-only set is refused", async () => { + await withNoNetwork(async () => { + await assert.rejects( + send(target, { + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { name: "createApiKey", arguments: {} }, + }), + /refusing to call tool "createApiKey"/ + ); + }); +}); + +test("a tools/call with no tool name is refused rather than passed through", async () => { + await withNoNetwork(async () => { + await assert.rejects( + send(target, { jsonrpc: "2.0", id: 3, method: "tools/call" }), + /refusing to call tool undefined/ + ); + }); +}); + +test("rpcCall is pinned: the generic escape hatch cannot carry an arbitrary method", async () => { + await withNoNetwork(async () => { + await assert.rejects( + send(target, { + jsonrpc: "2.0", + id: 4, + method: "tools/call", + params: { + name: "rpcCall", + arguments: { chain: "eth", method: "eth_sendRawTransaction" }, + }, + }), + /refusing rpcCall\("eth_sendRawTransaction"\)/ + ); + }); +}); + +test("the permitted read-only calls are not refused by the guard", async () => { + // The complement of the tests above. Without it, a guard that refused + // EVERYTHING would satisfy all of them, and the suite would be dead while + // reading as fully green. + const permitted = [ + { jsonrpc: "2.0" as const, id: 5, method: "tools/list" }, + { + jsonrpc: "2.0" as const, + id: 6, + method: "tools/call", + params: { name: "listChains", arguments: {} }, + }, + { + jsonrpc: "2.0" as const, + id: 7, + method: "tools/call", + params: { + name: "rpcCall", + arguments: { chain: "eth", method: "eth_blockNumber" }, + }, + }, + ]; + for (const body of permitted) { + const original = globalThis.fetch; + // Reaching the stub is the assertion: it means the guard passed the body on. + globalThis.fetch = (() => + Promise.reject(new Error("reached the network"))) as typeof fetch; + try { + await assert.rejects(send(target, body), /reached the network/); + } finally { + globalThis.fetch = original; + } + } +}); diff --git a/test/e2e/live-data-plane.e2e.ts b/test/e2e/live-data-plane.e2e.ts new file mode 100644 index 0000000..4f1c642 --- /dev/null +++ b/test/e2e/live-data-plane.e2e.ts @@ -0,0 +1,227 @@ +// Live data plane: the contracts a DEPLOYED instance must honour. +// +// Every assertion here is about the running service, not about this checkout — +// see test/e2e/live-parity.e2e.ts for the comparison between the two. Split on +// purpose: these are invariants that must hold on any healthy deployment, old +// or new, so a failure here is an outage or a regression, whereas a parity +// failure only means the deployed build is not this commit. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + resolveTarget, + send, + openSession, + errorOf, + resultText, + isToolError, + type LiveSession, +} from "./liveTarget.js"; + +const target = resolveTarget(); + +// One session for the read-only tool calls, so the suite takes a single slot out +// of the per-source session cap instead of one per test. +let shared: LiveSession | undefined; +const session = async (): Promise => + (shared ??= await openSession(target)); + +test.after(async () => { + await shared?.close(); +}); + +test("the deployed pod answers its health probe", async () => { + const res = await fetch(`${target.baseUrl}/healthz`, { + signal: AbortSignal.timeout(20_000), + }); + assert.equal(res.status, 200); + assert.deepEqual(await res.json(), { ok: true }); +}); + +test("initialize mints a session and identifies the server", async (t) => { + const s = await session(); + t.diagnostic(`target ${target.dataUrl} session ${s.id}`); + assert.match( + s.id, + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, + "the session id should be the randomUUID the transport is configured to mint" + ); + const info = s.initializeResult.serverInfo as + { name?: string; version?: string } | undefined; + assert.ok(info?.name, "initialize must identify the server"); + assert.ok(info.version, "initialize must state a version"); + assert.equal(s.initializeResult.protocolVersion, "2025-03-26"); +}); + +test("a request with no key is refused, and the refusal names how to send one", async () => { + const reply = await send(target, { + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "live e2e", version: "1" }, + }, + }); + assert.equal(reply.status, 401); + const err = errorOf(reply.message); + assert.equal(err?.code, -32001); + // Both carriers are named, because an agent that is told only "missing API + // key" has to guess which header to use, and guessing costs a round trip. + assert.match(String(err?.message), /x-ankr-api-key/); + assert.match(String(err?.message), /Bearer/i); +}); + +test("a live session id is not, by itself, authority to drive the session", async () => { + const s = await session(); + const reply = await send( + target, + { jsonrpc: "2.0", id: 99, method: "tools/list" }, + { sessionId: s.id } // deliberately no key + ); + assert.equal( + reply.status, + 401, + "a leaked session id with no credential must not be servable" + ); + assert.equal(errorOf(reply.message)?.code, -32001); +}); + +test("a session cannot be repointed at a different key", async () => { + const s = await session(); + // Never reaches an upstream: the bound-key check runs before anything is + // forwarded, so this string is compared against a fingerprint and dropped. + const reply = await send( + target, + { jsonrpc: "2.0", id: 98, method: "tools/list" }, + { sessionId: s.id, apiKey: "not-the-bound-key-000000000000000" } + ); + assert.equal(reply.status, 401); + assert.equal(errorOf(reply.message)?.code, -32001); + // The RULE only. Whether the refusal also names the remedy is a property of + // which build is deployed, not of the rule, so it is checked against this + // checkout in live-parity.e2e.ts instead of being transcribed here. + assert.match(String(errorOf(reply.message)?.message), /different API key/i); +}); + +test("an unknown session id is refused as a session problem, not a 500", async () => { + const reply = await send( + target, + { jsonrpc: "2.0", id: 97, method: "tools/list" }, + { + apiKey: target.apiKey, + sessionId: "00000000-0000-4000-8000-000000000000", + } + ); + assert.equal(reply.status, 400); + assert.equal(errorOf(reply.message)?.code, -32000); + assert.match(String(errorOf(reply.message)?.message), /initialize/i); +}); + +test("tools/list is served and every tool declares a strict input schema", async (t) => { + const s = await session(); + const reply = await s.call("tools/list"); + const tools = (reply.result as { tools?: Record[] }).tools; + assert.ok(tools && tools.length > 0, "a data session must advertise tools"); + t.diagnostic(`${String(tools.length)} tools advertised`); + for (const tool of tools) { + const schema = tool.inputSchema as + { type?: string; additionalProperties?: boolean } | undefined; + assert.equal( + schema?.type, + "object", + `${String(tool.name)} must advertise an object input schema` + ); + assert.equal( + schema.additionalProperties, + false, + `${String(tool.name)} must advertise that unknown arguments are rejected` + ); + assert.ok( + String(tool.description ?? "").length > 0, + `${String(tool.name)} must carry a description` + ); + } +}); + +test("an unknown argument is rejected by the deployed build, not silently dropped", async () => { + const s = await session(); + // The advertised `additionalProperties: false` is a claim about behaviour that + // a schema which merely STRIPS unknown keys would serialize identically, so + // the claim is checked by calling, not by reading the schema. + const reply = await s.call("tools/call", { + name: "listChains", + arguments: { thisArgumentDoesNotExist: 1 }, + }); + assert.ok( + isToolError(reply), + "a misspelled argument must be reported, not ignored" + ); +}); + +test("listChains answers from the deployed build", async () => { + const s = await session(); + const reply = await s.call("tools/call", { + name: "listChains", + arguments: {}, + }); + assert.ok(!isToolError(reply), `listChains failed: ${resultText(reply)}`); + const payload = JSON.parse(resultText(reply)) as { + aapiChains?: string[]; + aapiCount?: number; + }; + assert.ok( + payload.aapiChains?.includes("eth"), + "the Advanced API chain list must include eth" + ); + assert.equal( + payload.aapiCount, + payload.aapiChains?.length, + "the advertised count must match the list it counts" + ); +}); + +test("the deployed pod actually reaches a chain, not just its own process", async (t) => { + const s = await session(); + const reply = await s.call("tools/call", { + name: "rpcCall", + arguments: { chain: "eth", method: "eth_blockNumber", params: [] }, + }); + assert.ok(!isToolError(reply), `rpcCall failed: ${resultText(reply)}`); + const text = resultText(reply); + // The value is the point: a canned or cached answer would not track head. + const match = /0x[0-9a-fA-F]+|\b\d{6,}\b/.exec(text); + assert.ok(match, `no block number in the reply: ${text.slice(0, 300)}`); + const height = match[0].startsWith("0x") + ? Number.parseInt(match[0], 16) + : Number(match[0]); + t.diagnostic(`eth head as served: ${String(height)}`); + // Ethereum passed 21M blocks in 2024; anything below that is not a live head. + assert.ok( + height > 21_000_000, + `eth head ${String(height)} is not a plausible live height` + ); +}); + +test("a session the caller deletes is really gone", async () => { + const s = await openSession(target); + const del = await send(target, undefined, { + apiKey: target.apiKey, + sessionId: s.id, + method: "DELETE", + }); + assert.ok( + del.status < 300, + `DELETE returned ${String(del.status)}: ${del.bodyText.slice(0, 200)}` + ); + const after = await send( + target, + { jsonrpc: "2.0", id: 96, method: "tools/list" }, + { apiKey: target.apiKey, sessionId: s.id } + ); + assert.equal( + after.status, + 400, + "a deleted session must stop being servable, or teardown is cosmetic" + ); +}); diff --git a/test/e2e/live-mgmt-plane.e2e.ts b/test/e2e/live-mgmt-plane.e2e.ts new file mode 100644 index 0000000..3ed8e74 --- /dev/null +++ b/test/e2e/live-mgmt-plane.e2e.ts @@ -0,0 +1,234 @@ +// Live management plane: the OAuth posture a deployed instance must present. +// +// Nothing here authenticates and nothing here calls a management tool. The +// management surface is all writes and money, so this file probes only what an +// UNauthenticated caller sees: that the gate is closed, and that the discovery +// documents an MCP client needs in order to open it legitimately are present, +// well-formed and self-consistent. +// +// Why that is worth a test at all: a client cannot start the auth flow from a +// bare 401. It needs `WWW-Authenticate` to point at protected-resource metadata +// (RFC 9728), that document to name an authorization server, and that server's +// metadata (RFC 8414) to advertise the endpoints and PKCE method. Any one of +// those three missing leaves the plane technically "up" and practically +// unusable, and the in-process tests cannot see the ingress that serves them. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { resolveTarget, send } from "./liveTarget.js"; + +const target = resolveTarget(); +const skip = target.mgmtEnabled + ? false + : "E2E_MGMT=0 — target serves no management plane"; + +const getJson = async ( + url: string +): Promise<{ status: number; body: Record }> => { + const res = await fetch(url, { signal: AbortSignal.timeout(20_000) }); + const text = await res.text(); + let body: Record = {}; + try { + body = JSON.parse(text) as Record; + } catch { + body = { _unparsed: text.slice(0, 200) }; + } + return { status: res.status, body }; +}; + +test( + "an unauthenticated management request is refused and points at its metadata", + { skip }, + async () => { + const reply = await send( + target, + { + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "live e2e", version: "1" }, + }, + }, + { url: target.mgmtUrl } + ); + assert.equal(reply.status, 401); + const challenge = reply.headers.get("www-authenticate"); + assert.ok( + challenge, + "a 401 with no WWW-Authenticate gives a client nowhere to go" + ); + assert.match(challenge, /^Bearer/); + assert.match( + challenge, + /resource_metadata="[^"]+"/, + "the challenge must carry the protected-resource metadata URL (RFC 9728)" + ); + } +); + +test( + "the advertised protected-resource metadata resolves and describes THIS endpoint", + { skip }, + async (t) => { + const reply = await send( + target, + { + jsonrpc: "2.0", + id: 2, + method: "initialize", + params: { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "live e2e", version: "1" }, + }, + }, + { url: target.mgmtUrl } + ); + const advertised = /resource_metadata="([^"]+)"/.exec( + reply.headers.get("www-authenticate") ?? "" + )?.[1]; + assert.ok(advertised, "no resource_metadata URL to follow"); + t.diagnostic(`resource metadata: ${advertised}`); + + // Following the URL the server itself advertised, rather than one this test + // constructs, is the point: a document that exists at the path the spec + // suggests is worthless if the challenge points somewhere else. + const { status, body } = await getJson(advertised); + assert.equal(status, 200, "the advertised metadata URL must resolve"); + assert.equal( + body.resource, + target.mgmtUrl, + "the metadata must describe the endpoint that pointed at it" + ); + const servers = body.authorization_servers as string[] | undefined; + assert.ok( + servers && servers.length > 0, + "metadata naming no authorization server cannot start an auth flow" + ); + } +); + +test( + "each advertised authorization server publishes a usable, self-consistent metadata document", + { skip }, + async (t) => { + const reply = await send( + target, + { + jsonrpc: "2.0", + id: 3, + method: "initialize", + params: { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "live e2e", version: "1" }, + }, + }, + { url: target.mgmtUrl } + ); + const advertised = /resource_metadata="([^"]+)"/.exec( + reply.headers.get("www-authenticate") ?? "" + )?.[1]; + assert.ok(advertised); + const resourceMeta = await getJson(advertised); + const servers = (resourceMeta.body.authorization_servers ?? []) as string[]; + assert.ok(servers.length > 0); + + for (const issuer of servers) { + const metadataUrl = `${issuer.replace(/\/+$/, "")}/.well-known/oauth-authorization-server`; + const { status, body } = await getJson(metadataUrl); + assert.equal(status, 200, `${metadataUrl} must resolve`); + assert.equal(body.issuer, issuer, `${metadataUrl}: issuer must match`); + + const endpoints = [ + "authorization_endpoint", + "token_endpoint", + "registration_endpoint", + ] as const; + for (const name of endpoints) { + const value = body[name]; + assert.equal( + typeof value, + "string", + `${metadataUrl}: ${name} must be advertised` + ); + const url = new URL(String(value)); + assert.equal(url.protocol, "https:", `${name} must be https`); + assert.equal( + url.origin, + new URL(issuer).origin, + `${name} must live on the issuer's origin` + ); + } + + // Advertised is not the same as served, and only a live target can show + // the difference — but only the authorization endpoint can be probed this + // way. `/token` and `/register` are POST-only routes, so express answers a + // GET or HEAD on them with 404 whether or not they exist: probing those + // would assert nothing and fail on a healthy deployment (it did). Sending + // a real POST is not an option either, since registration is a write. + const authorizeUrl = new URL(String(body.authorization_endpoint)); + const probe = await fetch(authorizeUrl, { + redirect: "manual", + signal: AbortSignal.timeout(20_000), + }); + assert.notEqual( + probe.status, + 404, + `authorization_endpoint (${authorizeUrl.toString()}) is advertised but not served` + ); + + const pkce = body.code_challenge_methods_supported as + string[] | undefined; + assert.ok( + pkce?.includes("S256"), + `${metadataUrl}: S256 PKCE must be advertised` + ); + t.diagnostic(`${issuer}: PKCE ${(pkce ?? []).join(",")}`); + } + } +); + +test("a data-plane API key is not management authority", { skip }, async () => { + // The two planes share a host, and a raw Ankr key sent to the management path + // fails in a way that reads like a data-plane auth error. Pinning it here + // keeps the distinction observable: this is the WRONG SERVER answering, and + // anyone probing /mcp to check data-plane behaviour is measuring nothing. + const reply = await send( + target, + { + jsonrpc: "2.0", + id: 4, + method: "initialize", + params: { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "live e2e", version: "1" }, + }, + }, + { url: target.mgmtUrl, apiKey: target.apiKey } + ); + assert.equal( + reply.status, + 401, + "a raw data-plane key must never be accepted by the management plane" + ); + // The shape of the refusal is the tell. The data plane answers a bad + // credential with a JSON-RPC error OBJECT (`error.code === -32001`); this is a + // flat OAuth error STRING, which is how you know a different server replied. + assert.equal( + typeof reply.message?.error, + "string", + "the management plane answers with an OAuth error, not a JSON-RPC error object" + ); + assert.equal(reply.message?.error, "invalid_token"); +}); + +test("the host root exposes no application surface", { skip }, async () => { + const res = await fetch(`${target.baseUrl}/`, { + signal: AbortSignal.timeout(20_000), + }); + assert.equal(res.status, 404); +}); diff --git a/test/e2e/live-parity.e2e.ts b/test/e2e/live-parity.e2e.ts new file mode 100644 index 0000000..d9c263f --- /dev/null +++ b/test/e2e/live-parity.e2e.ts @@ -0,0 +1,329 @@ +// Is the DEPLOYED build the one in this checkout? +// +// The rest of the suite asserts invariants that any healthy deployment honours. +// This file asserts something narrower and more useful before a release: that +// the surface the live service advertises is byte-for-byte the surface this +// commit produces. It is the one check that catches "the branch is green, the +// pod is running last week's image" — a state every other gate in this repo, +// being in-process, is structurally blind to. +// +// The reference side is built by calling `createServer` from src/ over an +// in-memory transport, so the expectation is GENERATED from the code under +// review rather than transcribed into this file. A hand-copied expectation +// would drift from src/ and start asserting a fossil. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createServer as createNodeServer, type Server } from "node:http"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { createServer } from "../../src/server.js"; +import { createHttpApp } from "../../src/http.js"; +import { + resolveTarget, + openSession, + send, + errorOf, + type LiveSession, +} from "./liveTarget.js"; + +const target = resolveTarget(); + +// The fields that make up the advertised contract. Compared on both sides after +// the same normalization, so a difference is a real difference in what an agent +// sees and not an artifact of one side being parsed by the SDK and the other +// read as raw JSON. +interface ToolFacts { + name: string; + description: string; + inputSchema: unknown; + annotations: unknown; +} + +const normalize = (tools: Record[]): ToolFacts[] => + tools + .map((t) => ({ + name: String(t.name), + description: String(t.description ?? ""), + inputSchema: t.inputSchema ?? null, + annotations: t.annotations ?? null, + })) + .sort((a, b) => a.name.localeCompare(b.name)); + +interface Reference { + serverInfo: { name?: string; version?: string }; + instructions: string | undefined; + tools: ToolFacts[]; +} + +// The surface THIS checkout produces, obtained the same way a client would. +const buildReference = async (): Promise => { + const server = createServer("reference-key-not-used-for-tools-list"); + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "parity-reference", version: "1" }); + await server.connect(serverT); + await client.connect(clientT); + try { + const listed = await client.listTools(); + return { + serverInfo: client.getServerVersion() ?? {}, + instructions: client.getInstructions(), + tools: normalize(listed.tools as unknown as Record[]), + }; + } finally { + await client.close(); + } +}; + +// A second reference, for the contracts that are not in `tools/list`: the real +// express app from this checkout, on loopback. Error wording is part of the +// agent-facing contract too, and the only way to compare it without copying the +// expected sentence into this file (where it would rot) is to ask this checkout +// for it. +interface LocalApp { + baseUrl: string; + close: () => void; +} + +const startLocalApp = async (): Promise => { + // Bind first, pin the host allowlist, then build: the app resolves its whole + // posture once at construction and never re-reads process.env per request, so + // a variable set afterwards would not be seen (same ordering as + // test/data-http-session.test.ts). + const server = createNodeServer(); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + resolve(); + }); + }); + const { port } = server.address() as { port: number }; + const saved = process.env.MCP_ALLOWED_HOSTS; + process.env.MCP_ALLOWED_HOSTS = `127.0.0.1:${String(port)}`; + server.on("request", createHttpApp()); + return { + baseUrl: `http://127.0.0.1:${String(port)}`, + close: () => { + if (saved === undefined) delete process.env.MCP_ALLOWED_HOSTS; + else process.env.MCP_ALLOWED_HOSTS = saved; + server.close(); + }, + }; +}; + +// Drives the bound-key refusal against one target and returns the message it +// answers with. Identical request sequence on both sides, so a difference in the +// reply is a difference in the build. +const boundKeyRefusal = async ( + url: string, + key: string +): Promise => { + const localTarget = { ...target, dataUrl: url, apiKey: key }; + const init = await send( + localTarget, + { + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "parity", version: "1" }, + }, + }, + { apiKey: key } + ); + const sid = init.headers.get("mcp-session-id"); + assert.ok(sid, `no session minted by ${url}`); + const refusal = await send( + localTarget, + { jsonrpc: "2.0", id: 2, method: "tools/list" }, + { sessionId: sid, apiKey: "a-different-key-000000000000000000" } + ); + assert.equal(refusal.status, 401, `${url} did not refuse a rebind`); + return errorOf(refusal.message)?.message; +}; + +let live: LiveSession | undefined; +let local: LocalApp | undefined; +const liveSession = async (): Promise => + (live ??= await openSession(target)); + +test.after(async () => { + await live?.close(); + local?.close(); +}); + +// SHARK-3606 put the build's commit into `serverInfo.version` as semver build +// metadata: `+`. `BUILD_COMMIT` is supplied at image build +// time, so THIS process computes a bare `` while a correctly built +// deployment answers `+`. Comparing the two strings directly, as +// the first version of this test did, therefore fails against exactly the +// deployments that get build identity RIGHT, and passes against one that +// dropped `--build-arg BUILD_COMMIT` — the check inverted. +// +// What is assertable here, and what each part catches: +// - the NAME, exactly: a renamed server is a different product; +// - the RELEASE part, exactly: the deployment is a build of this version; +// - the PRESENCE and SHAPE of the commit suffix. This is the SHARK-3606 +// property itself, and its absence is the exact regression that follows +// from dropping the build arg, which shipped once already and left the sha +// living only in the registry tag; +// - the suffix's VALUE, when the operator knows which sha they expect and +// says so in `E2E_EXPECT_COMMIT`. +// +// "Is the deployment this checkout" is not carried by this string at all, and +// cannot be: the commit that adds a test is by construction not the commit that +// was built. That question is carried by the tool surface, the schemas, the +// descriptions, the instructions and the error wording, all generated from +// `src/` by the tests below. +const BUILD_VERSION = /^(?[^+]+)\+(?[0-9a-f]{40})$/; + +test("the deployed server identifies itself as a build of this release", async (t) => { + const reference = await buildReference(); + const s = await liveSession(); + const info = s.initializeResult.serverInfo as { + name?: string; + version?: string; + }; + t.diagnostic( + `live ${String(info.name)} ${String(info.version)} vs checkout ` + + `${String(reference.serverInfo.name)} ${String(reference.serverInfo.version)}` + ); + + assert.equal(info.name, reference.serverInfo.name); + + const liveVersion = String(info.version ?? ""); + const parsed = BUILD_VERSION.exec(liveVersion)?.groups; + assert.ok( + parsed?.release !== undefined && parsed.commit !== undefined, + `the deployed build answers ${JSON.stringify(liveVersion)}, which carries ` + + `no commit. serverInfo.version must be "+<40-hex commit>" ` + + `(SHARK-3606); a bare release means the image was built without ` + + `--build-arg BUILD_COMMIT, so the only place the sha exists is the ` + + `registry tag.` + ); + + // The reference side carries a suffix too when this process happens to have + // BUILD_COMMIT set, so compare release to release on both sides rather than + // assuming the local one is bare. + const referenceRelease = String(reference.serverInfo.version ?? "").split( + "+" + )[0]; + assert.equal( + parsed.release, + referenceRelease, + `the deployed build is release ${parsed.release}, this checkout is ` + + `${String(referenceRelease)}` + ); + + const expected = process.env.E2E_EXPECT_COMMIT?.trim(); + if (expected === undefined || expected === "") { + t.diagnostic( + `deployed commit ${parsed.commit}; set E2E_EXPECT_COMMIT to pin it` + ); + return; + } + assert.equal( + parsed.commit, + expected, + `the deployment is serving ${parsed.commit}, E2E_EXPECT_COMMIT asked for ` + + `${expected}` + ); +}); + +test("the deployed server delivers this checkout's session instructions", async () => { + const reference = await buildReference(); + const s = await liveSession(); + const liveInstructions = s.initializeResult.instructions; + assert.equal( + typeof liveInstructions, + typeof reference.instructions, + liveInstructions === undefined + ? "the deployment returns NO instructions on initialize while this " + + "checkout does — the session contract is not being delivered to agents" + : "the deployment returns instructions this checkout does not" + ); + assert.equal(liveInstructions, reference.instructions); +}); + +test("the deployed tool set is exactly this checkout's tool set", async (t) => { + const reference = await buildReference(); + const s = await liveSession(); + const reply = await s.call("tools/list"); + const liveTools = normalize( + (reply.result as { tools: Record[] }).tools + ); + + const liveNames = liveTools.map((x) => x.name); + const referenceNames = reference.tools.map((x) => x.name); + const missing = referenceNames.filter((n) => !liveNames.includes(n)); + const extra = liveNames.filter((n) => !referenceNames.includes(n)); + t.diagnostic( + `live ${String(liveNames.length)} tools, checkout ${String(referenceNames.length)}` + ); + assert.deepEqual( + { missing, extra }, + { missing: [], extra: [] }, + `tools missing from the deployment: [${missing.join(", ")}]; ` + + `tools the deployment has that this checkout does not: [${extra.join(", ")}]` + ); +}); + +test("every deployed tool advertises this checkout's description and schema", async (t) => { + const reference = await buildReference(); + const s = await liveSession(); + const reply = await s.call("tools/list"); + const liveTools = normalize( + (reply.result as { tools: Record[] }).tools + ); + const liveByName = new Map(liveTools.map((x) => [x.name, x])); + + // Reported as one list rather than failing on the first mismatch: before a + // release the useful output is every difference, not the alphabetically first. + const differences: string[] = []; + for (const expected of reference.tools) { + const actual = liveByName.get(expected.name); + if (!actual) continue; // the tool-set test above owns this case + if (actual.description !== expected.description) { + differences.push( + `${expected.name}: description differs ` + + `(live ${String(actual.description.length)} chars, ` + + `checkout ${String(expected.description.length)} chars)` + ); + } + if ( + JSON.stringify(actual.inputSchema) !== + JSON.stringify(expected.inputSchema) + ) { + differences.push(`${expected.name}: inputSchema differs`); + } + if ( + JSON.stringify(actual.annotations) !== + JSON.stringify(expected.annotations) + ) { + differences.push(`${expected.name}: annotations differ`); + } + } + t.diagnostic( + `${String(differences.length)} tool contract differences between the ` + + `deployment and this checkout` + ); + assert.deepEqual(differences, []); +}); + +test("the deployed build refuses a session rebind in this checkout's words", async (t) => { + local ??= await startLocalApp(); + const expected = await boundKeyRefusal( + `${local.baseUrl}/rpc`, + "parity-local-key-AAAAAAAAAAAAAAAA" + ); + const actual = await boundKeyRefusal(target.dataUrl, target.apiKey); + t.diagnostic( + `live ${String(actual?.length)} chars, checkout ${String(expected?.length)} chars` + ); + assert.equal( + actual, + expected, + "the refusal an agent sees in production differs from the one this " + + "checkout produces — the deployed build predates the current wording" + ); +}); diff --git a/test/e2e/liveTarget.ts b/test/e2e/liveTarget.ts new file mode 100644 index 0000000..c307947 --- /dev/null +++ b/test/e2e/liveTarget.ts @@ -0,0 +1,313 @@ +// The shared client for the LIVE e2e suite: one place that knows how to reach a +// deployed instance, and the one place that decides what this suite is allowed +// to send it. +// +// WHY THIS SUITE EXISTS SEPARATELY FROM `pnpm test`. Everything under +// `test/*.test.ts` is in-process: `createServer` over `InMemoryTransport`, or a +// loopback express app, with `globalThis.fetch` replaced. Those tests prove the +// code in this checkout behaves; they cannot prove anything about the build that +// is actually serving mcp.ankr.com. The two answer different questions, and the +// gap between them is exactly where "green suite, wrong thing deployed" lives. +// So this suite is opt-in (`pnpm test:e2e`), is NOT part of the push gate, and +// its file glob (`test/e2e/*.e2e.ts`) is deliberately outside the runner glob +// used by `pnpm test` (`test/*.test.ts`). +// +// WHY IT REFUSES TO SKIP. A live suite that quietly turns into a no-op when a +// variable is unset is worse than no suite: it reports success for a run that +// asserted nothing. Missing configuration throws at import time, which node's +// runner reports as a failed file. +import assert from "node:assert/strict"; + +// A request that never comes back must fail the run, not stall it. Node's fetch +// has no default timeout and `--test-timeout` defaults to Infinity, so without +// this the suite would hang against an unhealthy pod instead of reporting it. +// Longer than the in-process harness's bound (10s) because these requests cross +// the public internet, an ingress and a real upstream node. +const REQUEST_TIMEOUT_MS = 20_000; + +export interface LiveTarget { + readonly baseUrl: string; + readonly dataUrl: string; + readonly mgmtUrl: string; + readonly apiKey: string; + readonly mgmtEnabled: boolean; +} + +const trimSlash = (s: string): string => s.replace(/\/+$/, ""); + +const required = (name: string): string => { + const v = process.env[name]; + if (!v) { + throw new Error( + `${name} is not set. The live e2e suite talks to a real deployment and ` + + `cannot assert anything without a credential, so it fails here rather ` + + `than skipping. Set ${name} (see README, "Live e2e").` + ); + } + return v; +}; + +export const resolveTarget = (): LiveTarget => { + const baseUrl = trimSlash(process.env.E2E_BASE_URL ?? "https://mcp.ankr.com"); + const dataPath = process.env.E2E_DATA_PATH ?? "/rpc"; + const mgmtPath = process.env.E2E_MGMT_PATH ?? "/mcp"; + return { + baseUrl, + dataUrl: `${baseUrl}${dataPath}`, + mgmtUrl: `${baseUrl}${mgmtPath}`, + apiKey: required("ANKR_RPC_KEY"), + // A target that serves only the data plane (a locally built container, say) + // has no management surface to probe. Off by explicit opt-out, not by + // guessing from a 404 — a 404 is also what a BROKEN mgmt route returns. + mgmtEnabled: process.env.E2E_MGMT !== "0", + }; +}; + +// --------------------------------------------------------------------------- +// The read-only guarantee, enforced rather than promised. +// +// This suite runs against PRODUCTION. "It only reads" has to be a property of +// the code, not a claim in a comment that the next test to be added silently +// breaks. Every request goes through `send` below, and `send` refuses any +// JSON-RPC method outside this list, and any `tools/call` naming a tool outside +// the second list. Adding a write means editing this file, in a diff a reviewer +// will see. +// --------------------------------------------------------------------------- +const ALLOWED_METHODS: ReadonlySet = new Set([ + "initialize", + "notifications/initialized", + "tools/list", + "tools/call", +]); + +// Reads with no side effect and no write path anywhere behind them. `rpcCall` is +// pinned to a single method by `assertReadOnly` below, not merely by convention: +// it is the generic escape hatch, so an unconstrained entry here would re-open +// everything this guard closes. +const ALLOWED_TOOLS: ReadonlySet = new Set([ + "listChains", + "rpcCall", + "getBlock", +]); + +const ALLOWED_RPC_METHODS: ReadonlySet = new Set([ + "eth_blockNumber", + "eth_chainId", +]); + +export interface JsonRpcRequest { + jsonrpc: "2.0"; + id?: number; + method: string; + params?: Record; +} + +const assertReadOnly = (body: JsonRpcRequest): void => { + if (!ALLOWED_METHODS.has(body.method)) { + throw new Error( + `live e2e: refusing to send "${body.method}" — this suite runs against a ` + + `real deployment and is limited to the read-only methods in ` + + `test/e2e/liveTarget.ts` + ); + } + if (body.method !== "tools/call") return; + + const params = body.params ?? {}; + const tool = params.name; + if (typeof tool !== "string" || !ALLOWED_TOOLS.has(tool)) { + throw new Error( + `live e2e: refusing to call tool ${JSON.stringify(tool)} — only ` + + `${[...ALLOWED_TOOLS].join(", ")} are permitted against a live target` + ); + } + if (tool !== "rpcCall") return; + + const args = (params.arguments ?? {}) as Record; + const rpcMethod = args.method; + if (typeof rpcMethod !== "string" || !ALLOWED_RPC_METHODS.has(rpcMethod)) { + throw new Error( + `live e2e: refusing rpcCall(${JSON.stringify(rpcMethod)}) — rpcCall is ` + + `the generic escape hatch, so it is pinned to ` + + `${[...ALLOWED_RPC_METHODS].join(", ")} here` + ); + } +}; + +export interface RawReply { + readonly status: number; + readonly headers: Headers; + readonly bodyText: string; + /** The single JSON-RPC message in the reply, from either encoding. */ + readonly message: Record | undefined; +} + +// A Streamable HTTP reply is JSON *or* SSE, at the server's discretion, and the +// deployed data plane answers `initialize` as `text/event-stream` today. Reading +// only one encoding would make this suite pass or fail on a transport detail +// rather than on behaviour, so both are parsed into the same shape. +const parseMessage = ( + contentType: string, + bodyText: string +): Record | undefined => { + const raw = contentType.includes("text/event-stream") + ? bodyText + .split("\n") + .filter((l) => l.startsWith("data:")) + .map((l) => l.slice("data:".length).trim()) + .find((l) => l.length > 0) + : bodyText.trim() || undefined; + if (raw === undefined) return undefined; + try { + return JSON.parse(raw) as Record; + } catch { + return undefined; + } +}; + +export interface SendOptions { + /** Omit to send no credential at all (the 401 cases). */ + readonly apiKey?: string; + readonly sessionId?: string; + readonly url?: string; + readonly method?: "POST" | "GET" | "DELETE"; + readonly extraHeaders?: Record; +} + +export const send = async ( + target: LiveTarget, + body: JsonRpcRequest | undefined, + opts: SendOptions = {} +): Promise => { + if (body) assertReadOnly(body); + const url = opts.url ?? target.dataUrl; + const headers: Record = { + Accept: "application/json, text/event-stream", + ...(body ? { "Content-Type": "application/json" } : {}), + ...(opts.apiKey ? { "x-ankr-api-key": opts.apiKey } : {}), + ...(opts.sessionId ? { "mcp-session-id": opts.sessionId } : {}), + ...opts.extraHeaders, + }; + let res: Response; + try { + res = await fetch(url, { + method: opts.method ?? "POST", + headers, + body: body ? JSON.stringify(body) : undefined, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + } catch (err) { + const name = (err as { name?: string }).name; + if (name === "TimeoutError" || name === "AbortError") { + throw new Error( + `live e2e: no response from ${url} within ${String(REQUEST_TIMEOUT_MS)}ms` + ); + } + throw err; + } + const bodyText = await res.text(); + return { + status: res.status, + headers: res.headers, + bodyText, + message: parseMessage(res.headers.get("content-type") ?? "", bodyText), + }; +}; + +export interface LiveSession { + readonly id: string; + readonly initializeResult: Record; + call: ( + method: string, + params?: Record + ) => Promise>; + close: () => Promise; +} + +let nextId = 1; + +/** + * Opens a real MCP session against the target and returns a handle that speaks + * JSON-RPC over it. The caller MUST close it; the data plane caps concurrent + * sessions per source address, so a suite that leaked sessions would start + * failing itself with 429s. + */ +export const openSession = async (target: LiveTarget): Promise => { + const init = await send( + target, + { + jsonrpc: "2.0", + id: nextId++, + method: "initialize", + params: { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "agent-rpc-mcp live e2e", version: "1" }, + }, + }, + { apiKey: target.apiKey } + ); + assert.equal( + init.status, + 200, + `initialize failed against ${target.dataUrl}: HTTP ${String(init.status)} ${init.bodyText.slice(0, 300)}` + ); + const id = init.headers.get("mcp-session-id"); + assert.ok( + id, + "initialize returned no Mcp-Session-Id; every later request in this suite depends on it" + ); + const result = (init.message?.result ?? {}) as Record; + + await send( + target, + { jsonrpc: "2.0", method: "notifications/initialized" }, + { apiKey: target.apiKey, sessionId: id } + ); + + return { + id, + initializeResult: result, + call: async (method, params) => { + const reply = await send( + target, + { jsonrpc: "2.0", id: nextId++, method, params }, + { apiKey: target.apiKey, sessionId: id } + ); + assert.equal( + reply.status, + 200, + `${method} returned HTTP ${String(reply.status)}: ${reply.bodyText.slice(0, 300)}` + ); + assert.ok(reply.message, `${method} returned no parseable JSON-RPC body`); + return reply.message; + }, + close: async () => { + await send(target, undefined, { + apiKey: target.apiKey, + sessionId: id, + method: "DELETE", + }); + }, + }; +}; + +/** The JSON-RPC error object of a reply, or undefined when it carried a result. */ +export const errorOf = ( + message: Record | undefined +): { code?: number; message?: string } | undefined => + message?.error as { code?: number; message?: string } | undefined; + +/** The concatenated text of a `tools/call` result, for content assertions. */ +export const resultText = (message: Record): string => { + const result = message.result as + | { content?: { type?: string; text?: string }[]; isError?: boolean } + | undefined; + return (result?.content ?? []) + .map((c) => c.text ?? "") + .join("\n") + .trim(); +}; + +export const isToolError = (message: Record): boolean => + ((message.result as { isError?: boolean } | undefined)?.isError ?? false) || + message.error !== undefined;