From 39b1d6a222d92bb5cde91c765bce7e700009634e Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 05:47:52 +0000 Subject: [PATCH] feat(vitest-evals): Pass case expectations to suite judges and add createTaskHarness `run(input, options)` now forwards per-case judge params such as `expected` and `expectedTools` to every automatic suite judge, and `toSatisfyJudge(...)` reuses them for the returned run unless matcher options override them. The suite `judgeThreshold` keeps its existing semantics: it is compared against the average score of all automatic judges, like the legacy `threshold`. `createTaskHarness(name, task)` wraps a plain task function that returns `string | { result, toolCalls }` and builds the transcript events itself, so legacy task functions can move to the harness API without hand-written adapters. `createHarness(...)` still rejects top-level `toolCalls`. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- README.md | 9 +- docs/architecture.md | 12 +- .../content/docs/docs/harnesses/custom.mdx | 21 ++ .../docs/src/content/docs/docs/judges.mdx | 41 +++ .../docs/docs/judges/structured-output.mdx | 2 +- .../content/docs/docs/judges/tool-call.mdx | 12 +- packages/vitest-evals/README.md | 74 +++++- packages/vitest-evals/src/harness.test.ts | 234 ++++++++++++++++++ packages/vitest-evals/src/harness.ts | 138 +++++++++++ packages/vitest-evals/src/index.ts | 54 +++- skills/vitest-evals/SKILL.md | 8 +- 11 files changed, 588 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index fa67eaf..0572b24 100644 --- a/README.md +++ b/README.md @@ -230,8 +230,13 @@ Harness-backed suites stay close to plain Vitest: when the harness output type includes `undefined` - judges own their prompt, rubric, and parsing; LLM-backed judges use `ctx.runJudge(...)` from a configured `judgeHarness` -- pass scenario-specific judge criteria explicitly to matcher options, or bind - suite-wide criteria on the judge instance +- pass scenario-specific judge criteria as `run(input, { expected, + expectedTools })` so automatic suite judges see them, or explicitly to + matcher options; bind suite-wide criteria on the judge instance +- suite-level `judgeThreshold` compares the average score of all automatic + judges, like the legacy `threshold` +- wrap a plain `(input) => string | { result, toolCalls }` task function with + `createTaskHarness(...)` instead of building transcript events by hand - reporter output separates application, judge, and combined token and `costUsd` usage from normalized runs. First-party harnesses attach native spans when provider/runtime data is available; `createHarness(...)` attaches a fallback diff --git a/docs/architecture.md b/docs/architecture.md index 03cc965..fb9e27b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -67,6 +67,11 @@ event stream is flat: - `type: "tool_call"` for a tool request - `type: "tool_result"` for a completed or failed tool execution +`createTaskHarness(...)` is a thin wrapper over `createHarness(...)` for plain +task functions that return `string | { result, toolCalls }`; it builds the +user, `tool_call`, `tool_result`, and assistant events itself, so +`createHarness(...)` keeps rejecting top-level `toolCalls`. + This is the reporter and judge contract. `createHarness(...)` may accept strict camelCase message-shaped inputs at its lightweight result boundary: assistant `toolCalls`, `role: "tool"` results keyed by `toolCallId`, or AI SDK-style @@ -244,14 +249,17 @@ For each eval test in a harness-backed suite: 5. The harness returns a `HarnessRun` with `result.output`, `result.session`, `usage`, `timings`, `artifacts`, and `errors`. 6. Core stores that run on `task.meta.harness` for the reporter. -7. Automatic suite-level judges run against the normalized run/session pair. +7. Automatic suite-level judges run against the normalized run/session pair, + plus any per-case judge params passed as `run(input, options)`. The suite + `judgeThreshold` is compared against the average of their scores. 8. Each judge-harness invocation is stored under its judge score as `judgeRuns`. 9. The eval test asserts on the same returned result and session. 10. The reporter renders the recorded metadata and includes judge usage in totals without re-executing either harness. Explicit `expect(result).toSatisfyJudge(...)` calls use the run's typed output -and reuse registered input, metadata, and harness context +and reuse registered input, `run(input, options)` judge params, metadata, and +harness context when `result` came from the fixture-backed `run(...)`. Inside an eval test, calls on registered output objects or session objects reuse that exact run context; other raw values fall back to the current test's most recent diff --git a/packages/docs/src/content/docs/docs/harnesses/custom.mdx b/packages/docs/src/content/docs/docs/harnesses/custom.mdx index 5d1fa8b..980ebd0 100644 --- a/packages/docs/src/content/docs/docs/harnesses/custom.mdx +++ b/packages/docs/src/content/docs/docs/harnesses/custom.mdx @@ -68,6 +68,27 @@ export const qaHarness = createHarness({ }); ``` +### Plain Task Functions + +When the entrypoint already returns its output and the tool calls it made, use +`createTaskHarness()` and skip the transcript. It records the user input, one +`tool_call` event per call (plus a `tool_result` when the call has a `result`), +and the assistant output. + +```ts title="evals/qaHarness.ts" +import { createTaskHarness } from "vitest-evals"; +import { answerQuestionWithTools } from "../src/questionFlow"; + +export const qaHarness = createTaskHarness("qa-app", async (input: string) => { + const { text, toolCalls } = await answerQuestionWithTools(input); + + return { result: text, toolCalls }; +}); +``` + +A task can also return the output string alone. Legacy +`(input) => Promise` task functions fit as-is. + Return a full `HarnessRun` only when you need complete control over the session, usage, timings, artifacts, traces, and errors. Full runs must already use canonical `session.events`; `createHarness()` is the preferred custom diff --git a/packages/docs/src/content/docs/docs/judges.mdx b/packages/docs/src/content/docs/docs/judges.mdx index 6a7db49..d75e895 100644 --- a/packages/docs/src/content/docs/docs/judges.mdx +++ b/packages/docs/src/content/docs/docs/judges.mdx @@ -74,5 +74,46 @@ describeEval( ); ``` +## Per-Case Expectations + +Suite-level judges also need case-specific expectations. Pass them as the +second `run(...)` argument. The harness never sees them; each automatic judge +receives them in its context, and `toSatisfyJudge(...)` reuses them for the +returned run unless matcher options override them. + +```ts title="evals/capital.eval.ts" +import { + describeEval, + StructuredOutputJudge, + ToolCallJudge, +} from "vitest-evals"; +import { qaHarness } from "./qaHarness"; + +describeEval( + "capital questions", + { + harness: qaHarness, + judges: [ToolCallJudge(), StructuredOutputJudge()], + judgeThreshold: 0.75, + }, + (it) => { + it.for([ + { + input: "What is the capital of France?", + expected: { answer: "Paris" }, + expectedTools: ["lookupCapital"], + }, + ])("$input", async ({ input, ...expectations }, { run }) => { + await run(input, expectations); + }); + }, +); +``` + +`judgeThreshold` compares the average score of all automatic judges, so one +judge can fall short while the case still passes. A `null` score counts as 0. +Use explicit `toSatisfyJudge(...)` assertions when each judge must pass on its +own threshold. + Keep deterministic judges close to the harness contract. Use custom judges for rubrics that need domain-specific scoring or reusable explanations. diff --git a/packages/docs/src/content/docs/docs/judges/structured-output.mdx b/packages/docs/src/content/docs/docs/judges/structured-output.mdx index eb87a19..20b76f6 100644 --- a/packages/docs/src/content/docs/docs/judges/structured-output.mdx +++ b/packages/docs/src/content/docs/docs/judges/structured-output.mdx @@ -5,7 +5,7 @@ editUrl: false --- `StructuredOutputJudge()` compares the harness output with expected values from -the judge config or matcher options. Use it for deterministic checks when the harness +the judge config, `run(input, { expected })`, or matcher options. Use it for deterministic checks when the harness returns a parsed domain object instead of raw provider text. ## Usage diff --git a/packages/docs/src/content/docs/docs/judges/tool-call.mdx b/packages/docs/src/content/docs/docs/judges/tool-call.mdx index 2ec1722..a376b17 100644 --- a/packages/docs/src/content/docs/docs/judges/tool-call.mdx +++ b/packages/docs/src/content/docs/docs/judges/tool-call.mdx @@ -25,8 +25,16 @@ describeEval("capital questions", { ## Arguments -Use matcher options when a case-specific tool argument is the important -assertion. +Pass case-specific expected tools as a `run(...)` option so the suite judge +checks them: + +```ts title="evals/capital.eval.ts" +await run("What is the capital of France?", { + expectedTools: [{ name: "lookupCapital", arguments: { country: "France" } }], +}); +``` + +Or use matcher options when the tool argument is a separate assertion. ```ts title="evals/capital.eval.ts" const result = await run("What is the capital of France?"); diff --git a/packages/vitest-evals/README.md b/packages/vitest-evals/README.md index f2dfd2b..144e2f2 100644 --- a/packages/vitest-evals/README.md +++ b/packages/vitest-evals/README.md @@ -43,10 +43,13 @@ workflow. not return traces themselves. Span attributes include typed OpenTelemetry GenAI semantic keys while still allowing provider-specific metadata -- scenario-specific judge criteria should live in `input` or explicit matcher - options, depending on whether the app or only the judge needs them +- scenario-specific judge criteria should live in `input` when the app needs + them; per-case judge expectations go in `run(input, { expected, + expectedTools })` or explicit matcher options when only the judge needs them - suite-level `judges` are optional and run automatically after each `run(...)` -- suite-level `judgeThreshold` controls fail-on-score for those automatic judges +- suite-level `judgeThreshold` controls fail-on-score for those automatic + judges; it compares the average score of all automatic judges (a `null` + score counts as 0) and defaults to 1 - every judge is a named object with `assess(ctx)` - every judge receives `JudgeContext` with typed `input`, typed `output`, the normalized run/session, and tool calls; `output` is only optional @@ -130,6 +133,33 @@ describeEval("refund agent", { harness }, (it) => { }); ``` +When suite-level judges need per-case expectations, pass them as the second +`run(...)` argument. The harness never sees them; every automatic judge +receives them in its context, and `toSatisfyJudge(...)` reuses them for the +returned run unless matcher options override them: + +```ts +describeEval( + "refund agent", + { + harness, + judges: [ToolCallJudge(), StructuredOutputJudge()], + judgeThreshold: 0.75, + }, + (it) => { + it.for([ + { + input: "Refund invoice inv_123", + expected: { status: "approved" }, + expectedTools: ["lookupInvoice", "createRefund"], + }, + ])("$input", async ({ input, ...expectations }, { run }) => { + await run(input, expectations); + }); + }, +); +``` + ## Terminal Reporting The terminal reporter has two eval report levels. Normal mode prints compact @@ -205,6 +235,40 @@ factory and a `Runner` or `runner` factory. The harness calls normalization and replay hooks when the app needs a custom entrypoint or structured domain output mapping. +## Plain Task Functions + +When the system under test is a function that returns its output and, +optionally, the tool calls it made, wrap it with `createTaskHarness(...)` +instead of building transcript events by hand. Legacy +`(input) => Promise` task functions fit as-is: + +```ts +import { createTaskHarness, describeEval, ToolCallJudge } from "vitest-evals"; + +describeEval( + "refund agent", + { + harness: createTaskHarness("refund-agent", async (input) => { + const { text, toolCalls } = await runRefundAgent(input); + return { result: text, toolCalls }; + }), + judges: [ToolCallJudge()], + }, + (it) => { + it("looks up the invoice", async ({ run }) => { + await run("Refund invoice inv_123", { + expectedTools: ["lookupInvoice"], + }); + }); + }, +); +``` + +The harness records the user input, one `tool_call` event per reported call +(plus a `tool_result` event when the call has a `result`), and the assistant +output. Use `createHarness(...)` when the app also needs to report usage, +traces, or a richer transcript. + ## Custom App Harnesses First-party harness packages are conveniences, not the only supported path. If @@ -331,8 +395,8 @@ Use `Harness.run(...)` for the application under test. Calling `ctx.harness.run(...)` from inside a judge runs the application a second time, so reserve that for judges that intentionally need a second execution. Put criteria on `input` when they are part of the scenario itself; pass -case-specific judge criteria through matcher options, or configure suite-wide -criteria on the judge instance. +case-specific judge criteria through `run(input, options)` or matcher options, +or configure suite-wide criteria on the judge instance. `createHarness(...)` lightweight results must return at least one normalized event, either directly as `events` or from strict camelCase `messages`. Stored diff --git a/packages/vitest-evals/src/harness.test.ts b/packages/vitest-evals/src/harness.test.ts index a9e3eb5..6404bc7 100644 --- a/packages/vitest-evals/src/harness.test.ts +++ b/packages/vitest-evals/src/harness.test.ts @@ -5,7 +5,9 @@ import { createJudge, createJudgeHarness, createHarness, + createTaskHarness, describeEval, + type EvalRun, failedSpans, type GenAiSemanticAttributeKey, getHarnessRunFromError, @@ -358,6 +360,158 @@ describeEval( }, ); +const refundTaskHarness = createTaskHarness( + "refund-task", + async (input: string) => ({ + result: JSON.stringify({ status: "approved", invoice: input }), + toolCalls: [ + { + name: "lookupInvoice", + arguments: { invoiceId: "inv_123" }, + result: { refundable: true }, + }, + ], + }), +); + +const structuredTaskHarness = createTaskHarness( + "structured-task", + async () => ({ result: { status: "approved" } }), +); +type _TaskHarnessInfersStringOutput = Expect< + Equal< + Awaited>["output"], + string + > +>; +type _TaskHarnessInfersResultOutput = Expect< + Equal< + Awaited>["output"], + { status: string } + > +>; + +const caseParamJudgeSpy = vi.fn( + async (_opts: JudgeOptions) => ({ + score: 1, + }), +); + +const caseParamJudge = createJudge("CaseParamJudge", caseParamJudgeSpy); + +async function assertRunOptionTypes(run: EvalRun) { + await run("Refund invoice inv_123", { + expected: { status: "approved" }, + expectedTools: ["lookupInvoice", { name: "createRefund", arguments: {} }], + expectedStatus: "approved", + }); + + // @ts-expect-error expectedTools must use the ToolCallJudge expected shape. + await run("Refund invoice inv_123", { expectedTools: [1] }); +} + +void assertRunOptionTypes; + +test("createTaskHarness records input, tool calls, and output as events", async () => { + const taskHarness = createTaskHarness( + "refund-task", + async (input: string) => ({ + result: `approved ${input}`, + toolCalls: [ + { + name: "lookupInvoice", + arguments: { invoiceId: "inv_123" }, + result: { refundable: true }, + }, + { name: "createRefund", arguments: "not an object" }, + ], + }), + ); + + const run = await taskHarness.run("inv_123", { + artifacts: {}, + setArtifact: vi.fn(), + }); + + expect(run.output).toBe("approved inv_123"); + expect(run.session.events).toEqual([ + { type: "message", role: "user", content: "inv_123" }, + { + type: "tool_call", + id: "call_0", + name: "lookupInvoice", + arguments: { invoiceId: "inv_123" }, + }, + { + type: "tool_result", + toolCallId: "call_0", + name: "lookupInvoice", + content: { refundable: true }, + }, + { type: "tool_call", id: "call_1", name: "createRefund" }, + { type: "message", role: "assistant", content: "approved inv_123" }, + ]); + expect(toolCalls(run)).toEqual([ + { + name: "lookupInvoice", + arguments: { invoiceId: "inv_123" }, + status: "ok", + result: { refundable: true }, + }, + { name: "createRefund", status: "pending" }, + ]); + expect(spansByKind(run, "run").map((span) => span.name)).toEqual([ + "refund-task", + ]); +}); + +test("createTaskHarness accepts a bare string result and passes the run context", async () => { + const taskHarness = createTaskHarness( + "echo-task", + (input: string, { setArtifact }) => { + setArtifact("seen", input); + return input.toUpperCase(); + }, + ); + const artifacts: HarnessContext["artifacts"] = {}; + + const run = await taskHarness.run("hello", { + artifacts, + setArtifact: (name, value) => { + artifacts[name] = value; + }, + }); + + expect(run.output).toBe("HELLO"); + expect(run.artifacts).toEqual({ seen: "hello" }); + expect(run.session.events).toEqual([ + { type: "message", role: "user", content: "hello" }, + { type: "message", role: "assistant", content: "HELLO" }, + ]); +}); + +test("createTaskHarness attaches a failed run when the task throws", async () => { + const taskHarness = createTaskHarness("broken-task", async () => { + throw new TypeError("task failed"); + }); + + let thrown: unknown; + try { + await taskHarness.run("Refund invoice inv_123", { + artifacts: {}, + setArtifact: vi.fn(), + }); + } catch (error) { + thrown = error; + } + + const run = getHarnessRunFromError(thrown); + expect(userMessages(run!).map((message) => message.content)).toEqual([ + "Refund invoice inv_123", + ]); + expect(run?.errors).toEqual([{ type: "TypeError", message: "task failed" }]); +}); + test("createHarness rejects removed top-level tool call shortcut", async () => { const lightweightHarness = createHarness({ name: "custom-app", @@ -946,6 +1100,86 @@ describeEval( }, ); +describeEval( + "harness mode with per-case run options", + { + harness: refundTaskHarness, + judges: [ToolCallJudge(), StructuredOutputJudge(), caseParamJudge], + }, + (it) => { + it("passes run options to automatic judges", async ({ run, task }) => { + caseParamJudgeSpy.mockClear(); + + await run("inv_123", { + expected: { status: "approved" }, + expectedTools: ["lookupInvoice"], + expectedStatus: "approved", + input: "ignored: core context fields win", + }); + + expect( + task.meta.eval?.scores.map(({ name, score }) => [name, score]), + ).toEqual([ + ["ToolCallJudge", 1], + ["StructuredOutputJudge", 1], + ["CaseParamJudge", 1], + ]); + expect(caseParamJudgeSpy).toHaveBeenCalledWith( + expect.objectContaining({ + input: "inv_123", + expectedStatus: "approved", + }), + ); + }); + }, +); + +describeEval( + "harness mode averages automatic judges against judgeThreshold", + { + harness: refundTaskHarness, + judges: [ToolCallJudge(), StructuredOutputJudge()], + judgeThreshold: 0.5, + }, + (it) => { + it("passes when the average meets the threshold", async ({ run, task }) => { + await run("inv_123", { + expected: { status: "approved" }, + expectedTools: ["createRefund"], + }); + + expect(task.meta.eval).toMatchObject({ + avgScore: 0.5, + scores: [ + { name: "ToolCallJudge", score: 0 }, + { name: "StructuredOutputJudge", score: 1 }, + ], + thresholdFailed: false, + }); + }); + }, +); + +describeEval( + "harness mode reuses run options in explicit judge matchers", + { harness: refundTaskHarness }, + (it) => { + it("lets matcher options override run options", async ({ run, task }) => { + const result = await run("inv_123", { + expectedTools: ["lookupInvoice"], + }); + + await expect(result).toSatisfyJudge(ToolCallJudge()); + await expect(result).toSatisfyJudge(ToolCallJudge(), { + expectedTools: ["createRefund"], + threshold: null, + }); + + expect(task.meta.eval?.scores.map(({ score }) => score)).toEqual([1, 0]); + }); + }, +); + describeEval("harness mode with explicit judge matcher", { harness }, (it) => { it("curries run-scoped options into explicit judge assessor calls", async ({ run, diff --git a/packages/vitest-evals/src/harness.ts b/packages/vitest-evals/src/harness.ts index f6a8f69..0dc1d6f 100644 --- a/packages/vitest-evals/src/harness.ts +++ b/packages/vitest-evals/src/harness.ts @@ -284,6 +284,63 @@ export type CreateHarnessOptions< ) => MaybePromise>; }; +/** + * Tool call reported by a plain task function. + * + * @example + * ```ts + * const call: TaskHarnessToolCall = { + * name: "lookupInvoice", + * arguments: { invoiceId: "inv_123" }, + * result: { refundable: true }, + * }; + * ``` + */ +export type TaskHarnessToolCall = { + /** Tool name as exposed to the agent or application runtime. */ + name: string; + /** Tool arguments. Kept only when they normalize to a JSON object. */ + arguments?: unknown; + /** Tool result. When present, a matching `tool_result` event is recorded. */ + result?: unknown; +}; + +/** + * Value returned by a plain task function: the output alone when it is a + * string, or `{ result, toolCalls }`. + * + * @example + * ```ts + * const result: TaskHarnessResult = { + * result: "Refund approved.", + * toolCalls: [{ name: "lookupInvoice", arguments: { invoiceId: "inv_123" } }], + * }; + * ``` + */ +export type TaskHarnessResult = + // NoInfer keeps TOutput inferred from `result` when the task returns an object. + | NoInfer> + | { + /** App-facing output for assertions and judges. */ + result: TOutput; + /** Tool calls in the order the task made them. */ + toolCalls?: TaskHarnessToolCall[]; + }; + +/** + * Plain task function adapted by `createTaskHarness(...)`. + * + * Legacy `(input) => Promise` task functions + * already match this shape. + */ +export type TaskHarnessFn< + TInput = string, + TOutput extends JsonValue | undefined = string, +> = ( + input: TInput, + context: HarnessContext, +) => MaybePromise>; + function isJsonPrimitive(value: unknown): value is JsonPrimitive { return ( value === null || @@ -506,6 +563,87 @@ export function createHarness< return harness; } +/** + * Creates a harness from a plain task function that returns its output and, + * optionally, the tool calls it made. + * + * The harness records the transcript itself: the user input, one `tool_call` + * event per reported call (plus a `tool_result` event when the call has a + * `result`), then the assistant output. Use `createHarness(...)` instead when + * the app needs to report usage, traces, or a richer transcript. + * + * @param name - Stable harness name used in reports. + * @param task - Task function that runs the system under test. + * + * @example + * ```ts + * import { createTaskHarness, describeEval, ToolCallJudge } from "vitest-evals"; + * + * describeEval( + * "refund agent", + * { + * harness: createTaskHarness("refund-agent", async (input) => { + * const { text, toolCalls } = await runRefundAgent(input); + * return { result: text, toolCalls }; + * }), + * judges: [ToolCallJudge()], + * }, + * (it) => { + * it("looks up the invoice", async ({ run }) => { + * await run("Refund invoice inv_123", { + * expectedTools: ["lookupInvoice"], + * }); + * }); + * }, + * ); + * ``` + */ +export function createTaskHarness< + TInput = string, + TOutput extends JsonValue | undefined = string, +>( + name: string, + task: TaskHarnessFn, +): Harness { + return createHarness({ + name, + run: async ({ input, ...context }) => { + const value = await task(input, context); + const { result: output, toolCalls: calls = [] } = + typeof value === "string" + ? { result: value as TOutput, toolCalls: [] } + : value; + + const events: TranscriptEvent[] = [ + { type: "message", role: "user", content: normalizeContent(input) }, + ]; + calls.forEach((call, index) => { + const id = `call_${index}`; + const args = toJsonValue(call.arguments); + events.push({ + type: "tool_call", + id, + name: call.name, + ...(isJsonRecord(args) ? { arguments: args } : {}), + }); + if (call.result !== undefined) { + events.push({ + type: "tool_result", + toolCallId: id, + name: call.name, + content: normalizeContent(call.result), + }); + } + }); + if (output !== undefined) { + events.push({ type: "message", role: "assistant", content: output }); + } + + return { output, events } as SimpleHarnessResult; + }, + }); +} + /** * Normalizes a lightweight harness result into the reporter-facing run shape. * diff --git a/packages/vitest-evals/src/index.ts b/packages/vitest-evals/src/index.ts index 290a697..ab18023 100644 --- a/packages/vitest-evals/src/index.ts +++ b/packages/vitest-evals/src/index.ts @@ -31,6 +31,7 @@ import type { } from "./judges/types"; import type { JudgeHarness } from "./judges/judgeHarness"; import { createRunJudge } from "./judges/judgeHarness"; +import type { ToolCallJudgeExpectedTool } from "./judges/toolCallJudge"; import { wrapText } from "./wrapText"; type RecordedJudgeResult = JudgeResult & { @@ -59,6 +60,7 @@ type RegisteredJudgeRunContext = { harness: Harness; input: unknown; judgeHarness?: JudgeHarness; + judgeParams: EvalRunOptions; run: HarnessRun; signal?: AbortSignal; }; @@ -123,19 +125,48 @@ export type EvalHarnessRun< }; }; +/** + * Per-case judge params passed as the second `run(input, options)` argument. + * + * The harness never sees these values. They are spread into the context of + * every automatic suite judge, and `toSatisfyJudge(...)` reuses them for the + * returned run unless matcher options override them. Core context fields such + * as `input`, `output`, `run`, and `toolCalls` always win over these params. + * + * @example + * ```ts + * await run("Refund invoice inv_123", { + * expected: { status: "approved" }, + * expectedTools: ["lookupInvoice", "createRefund"], + * }); + * ``` + */ +export type EvalRunOptions = { + /** Expected output read by judges such as `StructuredOutputJudge()` and `FactualityJudge()`. */ + expected?: unknown; + /** Expected tool calls read by `ToolCallJudge()`. */ + expectedTools?: ToolCallJudgeExpectedTool[]; + /** Extra params for custom judges. */ + [param: string]: unknown; +}; + /** * Explicit harness execution primitive exposed to each eval test. * * @example * ```ts * const result = await run("Refund invoice inv_123"); + * await run("Refund invoice inv_123", { expectedTools: ["lookupInvoice"] }); * ``` */ export type EvalRun< TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, THarness extends Harness = Harness, -> = (input: TInput) => Promise>; +> = ( + input: TInput, + options?: EvalRunOptions, +) => Promise>; /** * Fixture-backed Vitest context exposed inside `describeEval(...)` tests. @@ -193,7 +224,11 @@ export interface DescribeEvalOptions< judges?: Array>>; /** Optional judge-side harness used only by judges that call `ctx.runJudge(...)`. */ judgeHarness?: JudgeHarness; - /** Passing threshold for automatic suite-level judges. `null` disables fail-on-score. */ + /** + * Passing threshold for automatic suite-level judges, compared against the + * average score of all of them (a `null` score counts as 0). Defaults to 1. + * `null` disables fail-on-score. + */ judgeThreshold?: number | null; /** Skips the entire eval suite when the predicate returns true. */ skipIf?: () => boolean; @@ -339,7 +374,7 @@ const evalTest = test signal, task, }) => { - return async (input: unknown) => { + return async (input: unknown, judgeParams: EvalRunOptions = {}) => { const resolvedHarness = harness as Harness< unknown, JsonValue | undefined @@ -378,6 +413,7 @@ const evalTest = test resolvedHarness, input, explicitJudgeHarness, + judgeParams, signal, ); } @@ -397,6 +433,7 @@ const evalTest = test resolvedHarness, input, explicitJudgeHarness, + judgeParams, signal, ); } @@ -419,6 +456,7 @@ const evalTest = test resolvedHarness, input, explicitJudgeHarness, + judgeParams, signal, ); @@ -430,6 +468,7 @@ const evalTest = test resolvedHarness, input, judgeHarness, + judgeParams, run, signal, ); @@ -600,6 +639,7 @@ async function applyAutomaticJudges< harness: THarness, input: TInput, judgeHarness: JudgeHarness | undefined, + judgeParams: EvalRunOptions, run: HarnessRun, signal?: AbortSignal, ) { @@ -613,6 +653,7 @@ async function applyAutomaticJudges< (judgeRun) => judgeRuns.push(judgeRun), ); const judgeOptions = { + ...judgeParams, input, output: run.output, toolCalls: runToolCalls, @@ -674,12 +715,14 @@ function recordJudgeRunContext( harness: Harness, input: TInput, judgeHarness: JudgeHarness | undefined, + judgeParams: EvalRunOptions, signal?: AbortSignal, ) { const context = { harness, input, judgeHarness, + judgeParams, run, signal, }; @@ -838,6 +881,7 @@ function buildJudgeAssertionOptions< >; return { + ...registeredContext?.judgeParams, ...judgeParams, input: resolvedInput, output, @@ -1215,6 +1259,7 @@ export { createFailedHarnessRun, createGenAiUsageAttributes, createHarness, + createTaskHarness, ensureRunTrace, failedSpans, getHarnessRunFromError, @@ -1273,6 +1318,9 @@ export { type SimpleSpanRecord, type SimpleTranscriptInput, type SimpleTraceRecord, + type TaskHarnessFn, + type TaskHarnessResult, + type TaskHarnessToolCall, type TimingSummary, type ToolCall, type UsageSummary, diff --git a/skills/vitest-evals/SKILL.md b/skills/vitest-evals/SKILL.md index 3855783..2766ff6 100644 --- a/skills/vitest-evals/SKILL.md +++ b/skills/vitest-evals/SKILL.md @@ -40,8 +40,12 @@ Use the harness-backed API as the only authoring model. - Keep rubric text and parsing in the judge. Put provider calls in a `judgeHarness`, then call them through `ctx.runJudge(...)` so usage is recorded. - Put scenario-owned criteria on the input value. Put direct-check expected - values in Vitest case rows. Pass per-case judge criteria through explicit - matcher options, and suite-wide criteria through judge config. + values in Vitest case rows. Pass per-case judge criteria as + `run(input, { expected, expectedTools })` for automatic suite judges or + through explicit matcher options, and suite-wide criteria through judge + config. +- Wrap plain `(input) => string | { result, toolCalls }` task functions with + `createTaskHarness(...)`. - Custom judges should use `createJudge(...)` for stable reporter labels. ## Verification