From 50cdbb1654fa7f1a9da5d5578c198036a38d0ae9 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 04:20:37 +0000 Subject: [PATCH] fix(evals): Run the eval suite on vitest-evals 0.18 vitest-evals 0.4.0 imports `vitest/reporters`, which vitest 5 removed, so `eval:ci` failed at startup and no evals ran. vitest-evals 0.18.0 supports vitest 5 (peer `>=5 <6`). - Bump the catalog to ^0.18.0 and keep the eval package on vitest 5. Allow vitest-evals and its @vitest-evals/core and report-ui 0.18.0 deps past minimumReleaseAge, as they were published today. - Import the scorer-first API from `vitest-evals/legacy`, where 0.18 keeps it. - Remove the unused root vitest-evals dev dependency. - Set SENTRY_CLI_NO_TELEMETRY=1 for the scorer's `sentry mcp` process. Evals must not send telemetry, and it skips the lazy telemetry require that fails when the CLI runs from source. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- docs/testing/overview.md | 2 +- package.json | 3 +- .../src/evals/autofix.eval.ts | 2 +- .../src/evals/create-dsn.eval.ts | 2 +- .../src/evals/create-project.eval.ts | 2 +- .../src/evals/create-team.eval.ts | 2 +- .../src/evals/get-issue.eval.ts | 2 +- .../src/evals/get-sentry-resource.eval.ts | 2 +- .../src/evals/get-trace-details.eval.ts | 2 +- .../src/evals/list-dsns.eval.ts | 2 +- .../src/evals/list-issues.eval.ts | 2 +- .../src/evals/list-organizations.eval.ts | 2 +- .../src/evals/list-projects.eval.ts | 2 +- .../src/evals/list-releases.eval.ts | 2 +- .../src/evals/list-tags.eval.ts | 2 +- .../src/evals/list-teams.eval.ts | 2 +- .../src/evals/natural-language-search.eval.ts | 2 +- .../src/evals/search-docs.eval.ts | 2 +- ...rch-events-agent-attribute-context.eval.ts | 4 +- .../evals/search-events-agent-regex.eval.ts | 2 +- .../src/evals/search-events-agent.eval.ts | 4 +- .../evals/search-events-environments.eval.ts | 2 +- .../src/evals/search-events.eval.ts | 2 +- .../evals/search-issue-events-agent.eval.ts | 4 +- .../src/evals/search-issue-events.eval.ts | 2 +- .../src/evals/search-issues-agent.eval.ts | 4 +- .../src/evals/search-issues.eval.ts | 2 +- .../src/evals/update-issue.eval.ts | 2 +- .../src/evals/update-project.eval.ts | 2 +- .../src/evals/utils/structuredOutputScorer.ts | 2 +- .../src/evals/utils/toolPredictionScorer.ts | 5 +- pnpm-lock.yaml | 53 ++++++++++++++----- pnpm-workspace.yaml | 5 +- 33 files changed, 81 insertions(+), 51 deletions(-) diff --git a/docs/testing/overview.md b/docs/testing/overview.md index 7a9e456fd..4a7891969 100644 --- a/docs/testing/overview.md +++ b/docs/testing/overview.md @@ -275,7 +275,7 @@ expect(result.timestamp).toMatchInlineSnapshot(); // ❌ ### Eval Test Structure ```typescript -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { TaskRunner, Factuality } from "./utils"; describeEval("tool-name", { diff --git a/package.json b/package.json index 9b679d6cb..6dfcd829d 100644 --- a/package.json +++ b/package.json @@ -65,8 +65,7 @@ "tsx": "catalog:", "typescript": "catalog:", "valibot": "catalog:", - "vitest": "catalog:", - "vitest-evals": "catalog:" + "vitest": "catalog:" }, "simple-git-hooks": { "pre-commit": "pnpm run pre-commit:generated && pnpm exec lint-staged --concurrent false" diff --git a/packages/mcp-server-evals/src/evals/autofix.eval.ts b/packages/mcp-server-evals/src/evals/autofix.eval.ts index d6a4590c8..cbbf20409 100644 --- a/packages/mcp-server-evals/src/evals/autofix.eval.ts +++ b/packages/mcp-server-evals/src/evals/autofix.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("begin-issue-fix", { diff --git a/packages/mcp-server-evals/src/evals/create-dsn.eval.ts b/packages/mcp-server-evals/src/evals/create-dsn.eval.ts index 5fa59f61a..d7cf92177 100644 --- a/packages/mcp-server-evals/src/evals/create-dsn.eval.ts +++ b/packages/mcp-server-evals/src/evals/create-dsn.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("create-dsn", { diff --git a/packages/mcp-server-evals/src/evals/create-project.eval.ts b/packages/mcp-server-evals/src/evals/create-project.eval.ts index f551c7ded..1eebc6eec 100644 --- a/packages/mcp-server-evals/src/evals/create-project.eval.ts +++ b/packages/mcp-server-evals/src/evals/create-project.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("create-project", { diff --git a/packages/mcp-server-evals/src/evals/create-team.eval.ts b/packages/mcp-server-evals/src/evals/create-team.eval.ts index 2a789f505..777cd6608 100644 --- a/packages/mcp-server-evals/src/evals/create-team.eval.ts +++ b/packages/mcp-server-evals/src/evals/create-team.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("create-team", { diff --git a/packages/mcp-server-evals/src/evals/get-issue.eval.ts b/packages/mcp-server-evals/src/evals/get-issue.eval.ts index dad5f9aec..658ca9d75 100644 --- a/packages/mcp-server-evals/src/evals/get-issue.eval.ts +++ b/packages/mcp-server-evals/src/evals/get-issue.eval.ts @@ -1,4 +1,4 @@ -import { describeEval, ToolCallScorer } from "vitest-evals"; +import { describeEval, ToolCallScorer } from "vitest-evals/legacy"; import { FIXTURES, McpToolCallTaskRunner } from "./utils"; describeEval("get-issue", { diff --git a/packages/mcp-server-evals/src/evals/get-sentry-resource.eval.ts b/packages/mcp-server-evals/src/evals/get-sentry-resource.eval.ts index efe0e08ef..e9f2eeb6a 100644 --- a/packages/mcp-server-evals/src/evals/get-sentry-resource.eval.ts +++ b/packages/mcp-server-evals/src/evals/get-sentry-resource.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("get-sentry-resource", { diff --git a/packages/mcp-server-evals/src/evals/get-trace-details.eval.ts b/packages/mcp-server-evals/src/evals/get-trace-details.eval.ts index 49e783ac9..5fa3f2a1b 100644 --- a/packages/mcp-server-evals/src/evals/get-trace-details.eval.ts +++ b/packages/mcp-server-evals/src/evals/get-trace-details.eval.ts @@ -1,4 +1,4 @@ -import { describeEval, ToolCallScorer } from "vitest-evals"; +import { describeEval, ToolCallScorer } from "vitest-evals/legacy"; import { FIXTURES, McpToolCallTaskRunner } from "./utils"; describeEval("get-trace-details", { diff --git a/packages/mcp-server-evals/src/evals/list-dsns.eval.ts b/packages/mcp-server-evals/src/evals/list-dsns.eval.ts index ad9341666..2500148be 100644 --- a/packages/mcp-server-evals/src/evals/list-dsns.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-dsns.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("list-dsns", { diff --git a/packages/mcp-server-evals/src/evals/list-issues.eval.ts b/packages/mcp-server-evals/src/evals/list-issues.eval.ts index 64295d64c..373c9d4fe 100644 --- a/packages/mcp-server-evals/src/evals/list-issues.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-issues.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("list-issues", { diff --git a/packages/mcp-server-evals/src/evals/list-organizations.eval.ts b/packages/mcp-server-evals/src/evals/list-organizations.eval.ts index 826e53402..cc549fd39 100644 --- a/packages/mcp-server-evals/src/evals/list-organizations.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-organizations.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("list-organizations", { diff --git a/packages/mcp-server-evals/src/evals/list-projects.eval.ts b/packages/mcp-server-evals/src/evals/list-projects.eval.ts index 50c698034..1160d08ab 100644 --- a/packages/mcp-server-evals/src/evals/list-projects.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-projects.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("list-projects", { diff --git a/packages/mcp-server-evals/src/evals/list-releases.eval.ts b/packages/mcp-server-evals/src/evals/list-releases.eval.ts index bba7d48da..c0aef147a 100644 --- a/packages/mcp-server-evals/src/evals/list-releases.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-releases.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("list-releases", { diff --git a/packages/mcp-server-evals/src/evals/list-tags.eval.ts b/packages/mcp-server-evals/src/evals/list-tags.eval.ts index 3470c83c8..34981578f 100644 --- a/packages/mcp-server-evals/src/evals/list-tags.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-tags.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("list-tags", { diff --git a/packages/mcp-server-evals/src/evals/list-teams.eval.ts b/packages/mcp-server-evals/src/evals/list-teams.eval.ts index 3e598dbe0..0bc17940a 100644 --- a/packages/mcp-server-evals/src/evals/list-teams.eval.ts +++ b/packages/mcp-server-evals/src/evals/list-teams.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("list-teams", { diff --git a/packages/mcp-server-evals/src/evals/natural-language-search.eval.ts b/packages/mcp-server-evals/src/evals/natural-language-search.eval.ts index 1b6f32321..9ea93b06d 100644 --- a/packages/mcp-server-evals/src/evals/natural-language-search.eval.ts +++ b/packages/mcp-server-evals/src/evals/natural-language-search.eval.ts @@ -1,7 +1,7 @@ import { SentryApiService } from "@sentry/mcp-core/api-client"; import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; import { searchIssuesAgent } from "@sentry/mcp-core/tools/search-issues/agent"; -import { describeEval, ToolCallScorer } from "vitest-evals"; +import { describeEval, ToolCallScorer } from "vitest-evals/legacy"; import "../setup-env"; import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; diff --git a/packages/mcp-server-evals/src/evals/search-docs.eval.ts b/packages/mcp-server-evals/src/evals/search-docs.eval.ts index 2d9454dca..084c01bdc 100644 --- a/packages/mcp-server-evals/src/evals/search-docs.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-docs.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("search-docs", { diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts index 7c96acc5b..38a5283ac 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts @@ -1,5 +1,5 @@ -import { describeEval } from "vitest-evals"; -import { ToolCallScorer } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; +import { ToolCallScorer } from "vitest-evals/legacy"; import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; import { SentryApiService } from "@sentry/mcp-core/api-client"; import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts index 2c127f319..245e4453c 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-regex.eval.ts @@ -1,6 +1,6 @@ import { SentryApiService } from "@sentry/mcp-core/api-client"; import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; -import { describeEval, ToolCallScorer } from "vitest-evals"; +import { describeEval, ToolCallScorer } from "vitest-evals/legacy"; import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; import "../setup-env"; diff --git a/packages/mcp-server-evals/src/evals/search-events-agent.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent.eval.ts index b415cdca7..08683a53e 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent.eval.ts @@ -1,5 +1,5 @@ -import { describeEval } from "vitest-evals"; -import { ToolCallScorer } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; +import { ToolCallScorer } from "vitest-evals/legacy"; import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; import { SentryApiService } from "@sentry/mcp-core/api-client"; import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; diff --git a/packages/mcp-server-evals/src/evals/search-events-environments.eval.ts b/packages/mcp-server-evals/src/evals/search-events-environments.eval.ts index 8b6541fd8..c0fa9a9b1 100644 --- a/packages/mcp-server-evals/src/evals/search-events-environments.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-environments.eval.ts @@ -1,7 +1,7 @@ import { SentryApiService } from "@sentry/mcp-core/api-client"; import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; import { expect } from "vitest"; -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import "../setup-env"; import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; diff --git a/packages/mcp-server-evals/src/evals/search-events.eval.ts b/packages/mcp-server-evals/src/evals/search-events.eval.ts index 4af9831c1..891c53952 100644 --- a/packages/mcp-server-evals/src/evals/search-events.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; // Note: This eval requires OPENROUTER_API_KEY to be set in the environment diff --git a/packages/mcp-server-evals/src/evals/search-issue-events-agent.eval.ts b/packages/mcp-server-evals/src/evals/search-issue-events-agent.eval.ts index 0bdca99d1..b195cd320 100644 --- a/packages/mcp-server-evals/src/evals/search-issue-events-agent.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-issue-events-agent.eval.ts @@ -1,5 +1,5 @@ -import { describeEval } from "vitest-evals"; -import { ToolCallScorer } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; +import { ToolCallScorer } from "vitest-evals/legacy"; import { searchIssueEventsAgent } from "@sentry/mcp-core/tools/search-issue-events/agent"; import { SentryApiService } from "@sentry/mcp-core/api-client"; import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; diff --git a/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts b/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts index 0b1e118c8..de90883c0 100644 --- a/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-issue-events.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; // Note: This eval requires OPENROUTER_API_KEY to be set in the environment diff --git a/packages/mcp-server-evals/src/evals/search-issues-agent.eval.ts b/packages/mcp-server-evals/src/evals/search-issues-agent.eval.ts index 56622f257..d5bb58af8 100644 --- a/packages/mcp-server-evals/src/evals/search-issues-agent.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-issues-agent.eval.ts @@ -1,7 +1,7 @@ import { SentryApiService } from "@sentry/mcp-core/api-client"; import { searchIssuesAgent } from "@sentry/mcp-core/tools/search-issues/agent"; -import { describeEval } from "vitest-evals"; -import { ToolCallScorer } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; +import { ToolCallScorer } from "vitest-evals/legacy"; import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; import "../setup-env"; diff --git a/packages/mcp-server-evals/src/evals/search-issues.eval.ts b/packages/mcp-server-evals/src/evals/search-issues.eval.ts index 2de7b9491..d0da1de57 100644 --- a/packages/mcp-server-evals/src/evals/search-issues.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-issues.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; // Note: This eval requires OPENROUTER_API_KEY to be set in the environment diff --git a/packages/mcp-server-evals/src/evals/update-issue.eval.ts b/packages/mcp-server-evals/src/evals/update-issue.eval.ts index e5cb3174b..8ff652e3e 100644 --- a/packages/mcp-server-evals/src/evals/update-issue.eval.ts +++ b/packages/mcp-server-evals/src/evals/update-issue.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("update-issue", { diff --git a/packages/mcp-server-evals/src/evals/update-project.eval.ts b/packages/mcp-server-evals/src/evals/update-project.eval.ts index 23cf58dca..a2b9e435b 100644 --- a/packages/mcp-server-evals/src/evals/update-project.eval.ts +++ b/packages/mcp-server-evals/src/evals/update-project.eval.ts @@ -1,4 +1,4 @@ -import { describeEval } from "vitest-evals"; +import { describeEval } from "vitest-evals/legacy"; import { FIXTURES, NoOpTaskRunner, ToolPredictionScorer } from "./utils"; describeEval("update-project", { diff --git a/packages/mcp-server-evals/src/evals/utils/structuredOutputScorer.ts b/packages/mcp-server-evals/src/evals/utils/structuredOutputScorer.ts index 2a5c257fb..087a4a2dc 100644 --- a/packages/mcp-server-evals/src/evals/utils/structuredOutputScorer.ts +++ b/packages/mcp-server-evals/src/evals/utils/structuredOutputScorer.ts @@ -1,4 +1,4 @@ -import type { Score, ScoreFn, BaseScorerOptions } from "vitest-evals"; +import type { Score, ScoreFn, BaseScorerOptions } from "vitest-evals/legacy"; interface StructuredOutputScorerOptions extends BaseScorerOptions { expected?: Record; diff --git a/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts b/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts index 270e3ea0b..4379d84b9 100644 --- a/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts +++ b/packages/mcp-server-evals/src/evals/utils/toolPredictionScorer.ts @@ -39,6 +39,9 @@ async function getAvailableTools(): Promise { SENTRY_AUTH_TOKEN: "mocked-access-token", SENTRY_CONFIG_DIR: configDir, SENTRY_HOST: "sentry.io", + // Evals must not send CLI telemetry. This also skips the lazy + // `require("../telemetry.js")` in the CLI, which fails from source. + SENTRY_CLI_NO_TELEMETRY: "1", }, }); @@ -145,7 +148,7 @@ CRITICAL: The expected tools represent the actual realistic behavior for this sp * ```typescript * import { ToolPredictionScorer } from './utils/toolPredictionScorer'; * import { NoOpTaskRunner } from './utils/runner'; - * import { describeEval } from 'vitest-evals'; + * import { describeEval } from 'vitest-evals/legacy'; * * describeEval("Sentry issue search", { * data: async () => [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f33845c5..db7fa68aa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -190,8 +190,8 @@ catalogs: specifier: ^5.0.5 version: 5.0.5 vitest-evals: - specifier: ^0.4.0 - version: 0.4.0 + specifier: ^0.18.0 + version: 0.18.0 workers-mcp: specifier: 0.1.0-3 version: 0.1.0-3 @@ -267,9 +267,6 @@ importers: vitest: specifier: 'catalog:' version: 5.0.5(@opentelemetry/api@1.9.1)(@types/node@22.16.0)(@vitest/coverage-v8@5.0.5)(jsdom@28.1.0)(msw@2.10.2(@types/node@22.16.0)(typescript@5.9.3))(vite@8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.20.3)(yaml@2.9.0)) - vitest-evals: - specifier: 'catalog:' - version: 0.4.0(tinyrainbow@3.2.0)(vitest@5.0.5) devDependencies: '@types/json-schema': specifier: ^7.0.15 @@ -724,7 +721,7 @@ importers: version: 11.6.0(react-router@7.18.4(react-dom@19.2.8(react@19.2.8))(react@19.2.8))(react@19.2.8) agents: specifier: 'catalog:' - version: 0.32.0(@ai-sdk/provider@4.0.4)(@ai-sdk/react@3.0.232(react@19.2.8)(zod@4.6.7))(@babel/core@7.28.0)(@babel/runtime@7.28.4)(@modelcontextprotocol/client@2.4.0)(@modelcontextprotocol/server@2.4.0)(ai@6.0.64(zod@4.6.7))(chat@4.40.0(ai@6.0.64(zod@4.6.7))(zod@4.6.7))(react@19.2.8)(rolldown@1.2.14)(vite@6.4.5(@types/node@24.13.5)(jiti@2.7.0)(lightningcss@1.33.0)(tsx@4.23.13)(yaml@2.9.0))(zod@4.6.7) + version: 0.32.0(@ai-sdk/provider@4.0.4)(@ai-sdk/react@3.0.232(react@19.2.8)(zod@4.6.7))(@babel/core@7.28.0)(@babel/runtime@7.28.4)(@modelcontextprotocol/client@2.4.0)(@modelcontextprotocol/sdk@1.32.2(@cfworker/json-schema@4.1.1)(zod@4.6.7))(@modelcontextprotocol/server@2.4.0)(ai@6.0.64(zod@4.6.7))(chat@4.40.0(ai@6.0.64(zod@4.6.7))(zod@4.6.7))(react@19.2.8)(rolldown@1.2.14)(vite@6.4.5(@types/node@24.13.5)(jiti@2.7.0)(lightningcss@1.33.0)(tsx@4.23.13)(yaml@2.9.0))(zod@4.6.7) ai: specifier: 'catalog:' version: 6.0.64(zod@4.6.7) @@ -954,7 +951,7 @@ importers: version: 5.0.5(@opentelemetry/api@1.9.1)(@types/node@24.13.5)(@vitest/coverage-v8@5.0.5)(jsdom@28.1.0)(msw@2.10.2(@types/node@24.13.5)(typescript@5.9.3))(vite@8.3.3(@types/node@24.13.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0)) vitest-evals: specifier: 'catalog:' - version: 0.4.0(tinyrainbow@3.2.0)(vitest@5.0.5) + version: 0.18.0(ai@6.0.64(zod@4.6.7))(tinyrainbow@3.2.0)(vitest@5.0.5)(zod@4.6.7) zod: specifier: ^4.6.6 version: 4.6.7 @@ -5196,6 +5193,12 @@ packages: oxc-transform-react: optional: true + '@vitest-evals/core@0.18.0': + resolution: {integrity: sha512-i+tCWoRe+/2MtgcYfw9LaanoW6VXuUz5LjW42uzgMfMRY/XXgnZmtSaXQARkhRP6LR6Lz3UNUOpoSMKrKyQh8A==} + + '@vitest-evals/report-ui@0.18.0': + resolution: {integrity: sha512-cXFIyYiI5tLj7v93V+TDhTuINjZYOwrQ4qjF7H7EaSiD4N4AVVQbQS0XTw5NHeCaNguadFfAtDfWTpoBGj6exg==} + '@vitest/coverage-v8@5.0.5': resolution: {integrity: sha512-4KpkGdIrSqBihx/JyP/e+S+IKB98etg1XDQiLAxksLfcShnwi1VReGeOT9NI43hLHbknosKp/1CCLn9Zgcs8mg==} peerDependencies: @@ -9127,11 +9130,19 @@ packages: vite: optional: true - vitest-evals@0.4.0: - resolution: {integrity: sha512-tvKIc8sCtK7LZnSTFLh5C7BlDzSZhefKzCR68QNShVa7gkiepg7CZH8j3T6ZBWwIa5VgfmFkZ1Iv5NKzUpSfGQ==} + vitest-evals@0.18.0: + resolution: {integrity: sha512-vIT6hrIBwIc9eNDn9zK3SxBTY2iFmoJOdVsp5Dw4lxQRqCamMMe8Et2yr91DB5SfZqAHc+YOm8htcN0rATL/hA==} + hasBin: true peerDependencies: - tinyrainbow: '*' - vitest: '*' + ai: '>=4 <7' + tinyrainbow: '>=2 <4' + vitest: '>=5 <6' + zod: ^4.6.6 + peerDependenciesMeta: + ai: + optional: true + zod: + optional: true vitest@5.0.5: resolution: {integrity: sha512-cPODOMF8YnkiiYtDdHo6n4wWmBrOBBbOIYH2d33M4l3CdNn72h+qMmO6uXffUN6A5AZFXNND/d+krS9mgkXRfg==} @@ -13159,6 +13170,14 @@ snapshots: optionalDependencies: '@rolldown/plugin-babel': 0.2.3(@babel/core@7.28.0)(@babel/runtime@7.28.4)(rolldown@1.2.14)(vite@8.3.3(@types/node@24.13.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0)) + '@vitest-evals/core@0.18.0': + dependencies: + zod: 4.6.7 + + '@vitest-evals/report-ui@0.18.0': + dependencies: + '@vitest-evals/core': 0.18.0 + '@vitest/coverage-v8@5.0.5(vitest@5.0.5)': dependencies: '@bcoe/v8-coverage': 1.0.2 @@ -13282,7 +13301,7 @@ snapshots: dependencies: humanize-ms: 1.2.1 - agents@0.32.0(@ai-sdk/provider@4.0.4)(@ai-sdk/react@3.0.232(react@19.2.8)(zod@4.6.7))(@babel/core@7.28.0)(@babel/runtime@7.28.4)(@modelcontextprotocol/client@2.4.0)(@modelcontextprotocol/server@2.4.0)(ai@6.0.64(zod@4.6.7))(chat@4.40.0(ai@6.0.64(zod@4.6.7))(zod@4.6.7))(react@19.2.8)(rolldown@1.2.14)(vite@6.4.5(@types/node@24.13.5)(jiti@2.7.0)(lightningcss@1.33.0)(tsx@4.23.13)(yaml@2.9.0))(zod@4.6.7): + agents@0.32.0(@ai-sdk/provider@4.0.4)(@ai-sdk/react@3.0.232(react@19.2.8)(zod@4.6.7))(@babel/core@7.28.0)(@babel/runtime@7.28.4)(@modelcontextprotocol/client@2.4.0)(@modelcontextprotocol/sdk@1.32.2(@cfworker/json-schema@4.1.1)(zod@4.6.7))(@modelcontextprotocol/server@2.4.0)(ai@6.0.64(zod@4.6.7))(chat@4.40.0(ai@6.0.64(zod@4.6.7))(zod@4.6.7))(react@19.2.8)(rolldown@1.2.14)(vite@6.4.5(@types/node@24.13.5)(jiti@2.7.0)(lightningcss@1.33.0)(tsx@4.23.13)(yaml@2.9.0))(zod@4.6.7): dependencies: '@babel/plugin-proposal-decorators': 8.0.2(@babel/core@7.28.0) '@cfworker/json-schema': 4.1.1 @@ -13300,6 +13319,7 @@ snapshots: optionalDependencies: '@ai-sdk/provider': 4.0.4 '@ai-sdk/react': 3.0.232(react@19.2.8)(zod@4.6.7) + '@modelcontextprotocol/sdk': 1.32.2(@cfworker/json-schema@4.1.1)(zod@4.6.7) '@modelcontextprotocol/server': 2.4.0 ai: 6.0.64(zod@4.6.7) chat: 4.40.0(ai@6.0.64(zod@4.6.7))(zod@4.6.7) @@ -17885,10 +17905,15 @@ snapshots: optionalDependencies: vite: 8.3.3(@types/node@24.13.5)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0) - vitest-evals@0.4.0(tinyrainbow@3.2.0)(vitest@5.0.5): + vitest-evals@0.18.0(ai@6.0.64(zod@4.6.7))(tinyrainbow@3.2.0)(vitest@5.0.5)(zod@4.6.7): dependencies: + '@vitest-evals/core': 0.18.0 + '@vitest-evals/report-ui': 0.18.0 tinyrainbow: 3.2.0 - vitest: 5.0.5(@opentelemetry/api@1.9.1)(@types/node@22.16.0)(@vitest/coverage-v8@5.0.5)(jsdom@28.1.0)(msw@2.10.2(@types/node@22.16.0)(typescript@5.9.3))(vite@8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.20.3)(yaml@2.9.0)) + vitest: 5.0.5(@opentelemetry/api@1.9.1)(@types/node@24.13.5)(@vitest/coverage-v8@5.0.5)(jsdom@28.1.0)(msw@2.10.2(@types/node@24.13.5)(typescript@5.9.3))(vite@8.3.3(@types/node@24.13.5)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0)) + optionalDependencies: + ai: 6.0.64(zod@4.6.7) + zod: 4.6.7 vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@22.16.0)(@vitest/coverage-v8@5.0.5)(jsdom@28.1.0)(msw@2.10.2(@types/node@22.16.0)(typescript@5.9.3))(vite@8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.20.3)(yaml@2.9.0)): dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5290c4e9f..0b84f03e9 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -102,7 +102,7 @@ catalog: valibot: ^1.4.2 vite: ^6.4.5 vitest: ^5.0.5 - vitest-evals: ^0.4.0 + vitest-evals: ^0.18.0 workers-mcp: 0.1.0-3 wrangler: 4.156.1 zod: ^4.6.6 @@ -171,3 +171,6 @@ minimumReleaseAgeExclude: - miniflare@5.20261101.0-beta - wrangler@4.156.1 - "@sentry/symbolic@14.0.0-alpha.5" + - "@vitest-evals/core@0.18.0" + - "@vitest-evals/report-ui@0.18.0" + - vitest-evals@0.18.0