From ea6d1f838b0454dba4a86b1ad40e2417a31c65f0 Mon Sep 17 00:00:00 2001 From: Rui Conti Date: Mon, 27 Jul 2026 17:43:54 -0400 Subject: [PATCH] test(e2e): cover user auth after service trigger Signed-off-by: Rui Conti --- .../agent/channels/eve.ts | 38 ++++++++ .../agent/connections/clickstate.ts | 49 ++++++++++ .../agent/instructions.md | 1 + ...nt-principal-after-service-trigger.eval.ts | 91 +++++++++++++++++++ 4 files changed, 179 insertions(+) create mode 100644 e2e/fixtures/agent-openapi-swagger/agent/channels/eve.ts create mode 100644 e2e/fixtures/agent-openapi-swagger/agent/connections/clickstate.ts create mode 100644 e2e/fixtures/agent-openapi-swagger/evals/auth/current-principal-after-service-trigger.eval.ts diff --git a/e2e/fixtures/agent-openapi-swagger/agent/channels/eve.ts b/e2e/fixtures/agent-openapi-swagger/agent/channels/eve.ts new file mode 100644 index 000000000..5261846d5 --- /dev/null +++ b/e2e/fixtures/agent-openapi-swagger/agent/channels/eve.ts @@ -0,0 +1,38 @@ +import type { AuthFn } from "eve/channels/auth"; +import { eveChannel } from "eve/channels/eve"; +import type { SessionAuthContext } from "eve/context"; + +const SERVICE_AUTHORIZATION = "Bearer e2e-current-auth-service"; +const USER_AUTHORIZATION = "Bearer e2e-current-auth-user"; + +function fixtureUser(principalId: string): SessionAuthContext { + return { + attributes: {}, + authenticator: "e2e-fixture", + issuer: "e2e", + principalId, + principalType: "user", + subject: principalId, + }; +} + +const authenticateService: AuthFn = (request) => { + if (request.headers.get("authorization") !== SERVICE_AUTHORIZATION) return null; + return { + attributes: {}, + authenticator: "e2e-bearer", + principalId: "e2e-service", + principalType: "service", + }; +}; + +const authenticateUser: AuthFn = (request) => { + if (request.headers.get("authorization") !== USER_AUTHORIZATION) return null; + return fixtureUser("e2e-human"); +}; + +const authenticateDefaultUser: AuthFn = () => fixtureUser("e2e-default-user"); + +export default eveChannel({ + auth: [authenticateService, authenticateUser, authenticateDefaultUser], +}); diff --git a/e2e/fixtures/agent-openapi-swagger/agent/connections/clickstate.ts b/e2e/fixtures/agent-openapi-swagger/agent/connections/clickstate.ts new file mode 100644 index 000000000..f65b6360b --- /dev/null +++ b/e2e/fixtures/agent-openapi-swagger/agent/connections/clickstate.ts @@ -0,0 +1,49 @@ +import { + ConnectionAuthorizationRequiredError, + defineInteractiveAuthorization, + defineMcpClientConnection, +} from "eve/connections"; + +const CONNECTION_NAME = "clickstate"; +const CLICKSTATE_MCP_URL = "https://clickstate-mcp.vercel.sh/mcp"; + +const READ_ONLY_TOOLS = [ + "run_query", + "list_warehouses", + "search_schema", + "describe", + "describe_event_type", + "get_query_guide", + "resolve_project", + "find_owner_by_host", +]; + +export default defineMcpClientConnection({ + url: CLICKSTATE_MCP_URL, + description: + "ClickState (read-only): query Vercel's ClickHouse warehouses for operational investigations, " + + "including workflow runs, usage facts, telemetry, and runtime logs. Start with the matching " + + "query guide, use schema discovery when fields are not established, and keep queries time-bounded and limited.", + tools: { allow: READ_ONLY_TOOLS }, + // Mirror the user-scoped Connect lifecycle without making the e2e depend on + // Vercel OIDC, connector installation, or the external MCP server. + auth: defineInteractiveAuthorization({ + async getToken() { + throw new ConnectionAuthorizationRequiredError(CONNECTION_NAME); + }, + async startAuthorization({ principal }) { + if (principal.type !== "user") { + throw new Error(`${CONNECTION_NAME}: expected a user principal`); + } + return { + challenge: { + instructions: `CLICKSTATE_CURRENT_PRINCIPAL=user:${principal.id}`, + url: "https://example.com/eve-e2e/authorize", + }, + }; + }, + async completeAuthorization() { + return { token: "unused" }; + }, + }), +}); diff --git a/e2e/fixtures/agent-openapi-swagger/agent/instructions.md b/e2e/fixtures/agent-openapi-swagger/agent/instructions.md index d442b666b..d1eb17f81 100644 --- a/e2e/fixtures/agent-openapi-swagger/agent/instructions.md +++ b/e2e/fixtures/agent-openapi-swagger/agent/instructions.md @@ -1,3 +1,4 @@ # Identity You are a helpful assistant. Use the TfL connection whenever the user asks for Transport for London API data. +Use the ClickState connection whenever the user asks you to investigate ClickState. diff --git a/e2e/fixtures/agent-openapi-swagger/evals/auth/current-principal-after-service-trigger.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/auth/current-principal-after-service-trigger.eval.ts new file mode 100644 index 000000000..597a1a41a --- /dev/null +++ b/e2e/fixtures/agent-openapi-swagger/evals/auth/current-principal-after-service-trigger.eval.ts @@ -0,0 +1,91 @@ +import type { HandleMessageStreamEvent } from "eve/client"; +import { defineEval } from "eve/evals"; +import { equals } from "eve/evals/expect"; + +const CONNECTION = "clickstate"; +const SEARCH_TOOL = "connection_search"; +const SERVICE_COMPLETE = "AUTOMATED_INVESTIGATION_COMPLETE"; + +export default defineEval({ + description: + "A human can authorize a user-scoped connection after the service turn records a principal error.", + + async test(t) { + const serviceTurn = await t.send({ + headers: { authorization: "Bearer e2e-current-auth-service" }, + message: [ + "Automated trigger: investigate a synthetic deployment alert.", + "Call `connection_search` exactly once with connection `clickstate` and keywords `workflow runs`.", + "After the expected service-principal failure, do not retry or use another tool.", + `Reply with exactly: ${SERVICE_COMPLETE}`, + ].join("\n"), + }); + serviceTurn.expectOk(); + serviceTurn.calledTool(SEARCH_TOOL, { + count: 1, + input: { connection: CONNECTION }, + output: /active session is scoped to "service"/iu, + status: "failed", + }); + serviceTurn.notEvent("authorization.required", { + data: { name: CONNECTION }, + }); + serviceTurn.messageIncludes(SERVICE_COMPLETE); + serviceTurn.eventsSatisfy( + "the failed search result reaches a later model step before the service turn completes", + serviceFailurePrecedesCompletion, + ); + + const humanTurn = await t.send({ + headers: { authorization: "Bearer e2e-current-auth-user" }, + message: "@sre can u check clickstate bro?", + }); + humanTurn.expectOk(); + + await t.require(humanTurn.sessionId, equals(serviceTurn.sessionId)); + humanTurn.calledTool(SEARCH_TOOL, { + count: 1, + input: { connection: CONNECTION }, + status: "pending", + }); + humanTurn.event("authorization.required", { + count: 1, + data: { + authorization: { + instructions: "CLICKSTATE_CURRENT_PRINCIPAL=user:e2e-human", + }, + name: CONNECTION, + }, + }); + }, +}); + +function serviceFailurePrecedesCompletion(events: readonly HandleMessageStreamEvent[]): boolean { + const failureIndex = events.findIndex( + (event) => + event.type === "action.result" && + event.data.status === "failed" && + event.data.result.kind === "tool-result" && + event.data.result.toolName === SEARCH_TOOL, + ); + const failure = events[failureIndex]; + if (failure?.type !== "action.result") return false; + + const nextStepIndex = events.findIndex( + (event, eventIndex) => + eventIndex > failureIndex && + event.type === "step.started" && + event.data.stepIndex === failure.data.stepIndex + 1, + ); + if (nextStepIndex < 0) return false; + + return events + .slice(nextStepIndex + 1) + .some( + (event) => + event.type === "message.completed" && + event.data.stepIndex === failure.data.stepIndex + 1 && + event.data.finishReason !== "tool-calls" && + event.data.message === SERVICE_COMPLETE, + ); +}