From b6318d90bf5f7c165615427d62409de03d7c2dc5 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 23 Jul 2026 22:14:32 +0000 Subject: [PATCH 1/4] feat(eval): add eval evaluator CLI commands (CRUD + LLaJ/code-based) Add the imperative `agentcore eval evaluator` command surface: llm-as-a-judge create/update, code-based create/update, and type-agnostic get/list/delete. CLI-only for now; the TUI follows later. - New CoreEvalClient (consumer-owned interface) + EvalClient impl over the Bedrock AgentCore control plane. Update paths do get-then-merge since UpdateEvaluator replaces the whole evaluatorConfig union. - Rating scale accepts a preset (--rating-scale) or raw JSON (--rating-scale-json). - Source-aware field values: inline, file://, or - for stdin. - --timeout has no CLI default; the service applies its own. - list --type filters the returned page client-side (the API paginates only): Builtin | code-based | llm-as-a-judge. - delete requires --yes (headless-safe confirmation). Tests: handler behavior via TestCoreClient, EvalClient get-then-merge unit tests, and source resolver tests. --- README.md | 38 ++ src/core/eval.test.ts | 102 ++++++ src/core/eval.tsx | 138 +++++++ src/core/index.tsx | 2 + .../eval/evaluator/code-based/index.tsx | 89 +++++ src/handlers/eval/evaluator/delete/index.tsx | 28 ++ .../eval/evaluator/evaluator.test.tsx | 342 ++++++++++++++++++ src/handlers/eval/evaluator/get/index.tsx | 19 + src/handlers/eval/evaluator/index.tsx | 28 ++ src/handlers/eval/evaluator/list/index.tsx | 47 +++ .../eval/evaluator/llm-as-a-judge/index.tsx | 147 ++++++++ src/handlers/eval/index.tsx | 10 + src/handlers/eval/ratingScale.tsx | 50 +++ src/handlers/eval/source.test.ts | 52 +++ src/handlers/eval/source.tsx | 58 +++ src/handlers/eval/types.tsx | 62 ++++ src/handlers/index.tsx | 2 + src/handlers/root.test.tsx | 1 + src/handlers/types.tsx | 2 + src/testing/TestCoreClient.tsx | 87 +++++ src/testing/index.tsx | 1 + 21 files changed, 1305 insertions(+) create mode 100644 src/core/eval.test.ts create mode 100644 src/core/eval.tsx create mode 100644 src/handlers/eval/evaluator/code-based/index.tsx create mode 100644 src/handlers/eval/evaluator/delete/index.tsx create mode 100644 src/handlers/eval/evaluator/evaluator.test.tsx create mode 100644 src/handlers/eval/evaluator/get/index.tsx create mode 100644 src/handlers/eval/evaluator/index.tsx create mode 100644 src/handlers/eval/evaluator/list/index.tsx create mode 100644 src/handlers/eval/evaluator/llm-as-a-judge/index.tsx create mode 100644 src/handlers/eval/index.tsx create mode 100644 src/handlers/eval/ratingScale.tsx create mode 100644 src/handlers/eval/source.test.ts create mode 100644 src/handlers/eval/source.tsx create mode 100644 src/handlers/eval/types.tsx diff --git a/README.md b/README.md index b2a3be481..33a78604e 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,17 @@ agentcore # interactive TUI │ └── endpoint │ ├── get # get a Runtime endpoint by qualifier │ └── list # list a Runtime's endpoints +├── eval # evaluate and optimize AgentCore agents +│ └── evaluator # manage AgentCore evaluators +│ ├── llm-as-a-judge # LLM-as-a-Judge evaluators +│ │ ├── create # create (instructions + rating scale + model) +│ │ └── update # update (merged over the existing config) +│ ├── code-based # code-based (Lambda-backed) evaluators +│ │ ├── create # create (Lambda ARN + optional timeout) +│ │ └── update # update (merged over the existing config) +│ ├── get # get an evaluator by id (type-agnostic) +│ ├── list # list evaluators (client-side --type filter) +│ └── delete # delete an evaluator by id (requires --yes) └── config # read/write global config values ``` @@ -114,8 +125,35 @@ agentcore identity api-key-credential-provider get --name my-provider agentcore identity api-key-credential-provider list --max-results 10 agentcore identity api-key-credential-provider update --name my-provider --api-key agentcore identity api-key-credential-provider delete --name my-provider + +# Manage evaluators +# Create an LLM-as-a-Judge evaluator with a rating-scale preset. +agentcore eval evaluator llm-as-a-judge create \ + --name order-support-quality \ + --level SESSION \ + --model us.anthropic.claude-sonnet-4-5 \ + --instructions "Judge whether the order-support agent answered correctly." \ + --rating-scale 1-5-quality \ + --json + +# Create a code-based (Lambda-backed) evaluator; timeout defaults to the service value. +agentcore eval evaluator code-based create \ + --name refund-policy-compliance \ + --level SESSION \ + --lambda-arn arn:aws:lambda:us-west-2:123456789012:function:refund-policy \ + --json + +# Get, list, delete. --type filters the returned page (Builtin | code-based | llm-as-a-judge). +agentcore eval evaluator get --id --json +agentcore eval evaluator list --type llm-as-a-judge --max-results 20 --json +agentcore eval evaluator delete --id --yes --json ``` +Source-aware values: any field flag documented as such accepts the value inline, +`file://` to read it from a file, or `-` to read it from stdin (the AWS CLI +`file://` convention). A command reads stdin from at most one flag. For example, +`--instructions file://order-quality.txt` or `--instructions -`. + Bare Runtime branches and leaves require a TTY on stdin and stdout. Supplying operation flags runs the command headlessly, and `--json` always suppresses TUI rendering. diff --git a/src/core/eval.test.ts b/src/core/eval.test.ts new file mode 100644 index 000000000..43260d314 --- /dev/null +++ b/src/core/eval.test.ts @@ -0,0 +1,102 @@ +import { test, expect } from "bun:test"; +import { + GetEvaluatorCommand, + UpdateEvaluatorCommand, + type GetEvaluatorResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { EvalClient } from "./eval"; +import type { AwsClients, CoreOptions } from "./types"; + +const OPTIONS: CoreOptions = { region: "us-west-2", endpointUrl: undefined }; + +// fakeClients returns an AwsClients whose control().send dispatches GetEvaluator +// to `getResponse` and records UpdateEvaluator inputs into `updates`. Only +// control() is used by EvalClient; data()/iam() throw if ever reached. +function fakeClients(getResponse: GetEvaluatorResponse, updates: unknown[]): AwsClients { + const control = { + send: async (command: { input: unknown }) => { + if (command instanceof GetEvaluatorCommand) return getResponse; + if (command instanceof UpdateEvaluatorCommand) { + updates.push(command.input); + return { evaluatorId: "e-1" }; + } + throw new Error(`unexpected command ${command.constructor.name}`); + }, + }; + return { + control: () => control as never, + data: () => { + throw new Error("data client not expected"); + }, + iam: () => { + throw new Error("iam client not expected"); + }, + }; +} + +test("updateLlmAsAJudgeEvaluator preserves existing fields not passed", async () => { + const updates: unknown[] = []; + const current: GetEvaluatorResponse = { + evaluatorConfig: { + llmAsAJudge: { + instructions: "old instructions", + ratingScale: { numerical: [{ value: 1, label: "L", definition: "d" }] }, + modelConfig: { bedrockEvaluatorModelConfig: { modelId: "old-model" } }, + }, + }, + } as GetEvaluatorResponse; + + const client = new EvalClient(fakeClients(current, updates)); + // Only the model changes; instructions + ratingScale should be carried over. + await client.updateLlmAsAJudgeEvaluator("e-1", { model: "new-model" }, OPTIONS); + + expect(updates).toHaveLength(1); + const sent = updates[0] as any; + expect(sent.evaluatorConfig.llmAsAJudge.modelConfig.bedrockEvaluatorModelConfig.modelId).toBe( + "new-model", + ); + expect(sent.evaluatorConfig.llmAsAJudge.instructions).toBe("old instructions"); + expect(sent.evaluatorConfig.llmAsAJudge.ratingScale.numerical[0].label).toBe("L"); +}); + +test("updateLlmAsAJudgeEvaluator rejects a non-LLM evaluator", async () => { + const current: GetEvaluatorResponse = { + evaluatorConfig: { codeBased: { lambdaConfig: { lambdaArn: "arn:x" } } }, + } as GetEvaluatorResponse; + const client = new EvalClient(fakeClients(current, [])); + await expect( + client.updateLlmAsAJudgeEvaluator("e-1", { instructions: "x" }, OPTIONS), + ).rejects.toThrow(/not an LLM-as-a-Judge evaluator/); +}); + +test("updateCodeBasedEvaluator preserves the lambda ARN when only timeout changes", async () => { + const updates: unknown[] = []; + const current: GetEvaluatorResponse = { + evaluatorConfig: { + codeBased: { lambdaConfig: { lambdaArn: "arn:keep", lambdaTimeoutInSeconds: 10 } }, + }, + } as GetEvaluatorResponse; + + const client = new EvalClient(fakeClients(current, updates)); + await client.updateCodeBasedEvaluator("e-1", { timeout: 45 }, OPTIONS); + + const sent = updates[0] as any; + expect(sent.evaluatorConfig.codeBased.lambdaConfig.lambdaArn).toBe("arn:keep"); + expect(sent.evaluatorConfig.codeBased.lambdaConfig.lambdaTimeoutInSeconds).toBe(45); +}); + +test("updateCodeBasedEvaluator rejects a non-code-based evaluator", async () => { + const current = { + evaluatorConfig: { + llmAsAJudge: { + instructions: "i", + ratingScale: { numerical: [] }, + modelConfig: { bedrockEvaluatorModelConfig: { modelId: "m" } }, + }, + }, + } as unknown as GetEvaluatorResponse; + const client = new EvalClient(fakeClients(current, [])); + await expect(client.updateCodeBasedEvaluator("e-1", { timeout: 30 }, OPTIONS)).rejects.toThrow( + /not a code-based evaluator/, + ); +}); diff --git a/src/core/eval.tsx b/src/core/eval.tsx new file mode 100644 index 000000000..aadd2791f --- /dev/null +++ b/src/core/eval.tsx @@ -0,0 +1,138 @@ +import { + CreateEvaluatorCommand, + DeleteEvaluatorCommand, + GetEvaluatorCommand, + ListEvaluatorsCommand, + UpdateEvaluatorCommand, + type CreateEvaluatorRequest, + type CreateEvaluatorResponse, + type DeleteEvaluatorResponse, + type EvaluatorConfig, + type GetEvaluatorResponse, + type ListEvaluatorsResponse, + type UpdateEvaluatorResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; +import type { AwsClients, CoreOptions } from "./types"; +import { toClientConfig } from "./utils"; + +export class EvalClient implements CoreEvalClient { + constructor(private readonly clients: AwsClients) {} + + async createEvaluator( + request: CreateEvaluatorRequest, + options: CoreOptions, + ): Promise { + return this.clients.control(toClientConfig(options)).send(new CreateEvaluatorCommand(request)); + } + + // updateLlmAsAJudgeEvaluator rebuilds the full llmAsAJudge config from the + // current evaluator, overlays the provided fields, and sends it. UpdateEvaluator + // replaces the entire evaluatorConfig union, and the llmAsAJudge arm requires + // instructions + ratingScale + modelConfig together, so a partial update would + // otherwise drop the fields the caller didn't pass. + async updateLlmAsAJudgeEvaluator( + id: string, + update: LlmAsAJudgeUpdate, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); + const existing = + current.evaluatorConfig && "llmAsAJudge" in current.evaluatorConfig + ? current.evaluatorConfig.llmAsAJudge + : undefined; + + const instructions = update.instructions ?? existing?.instructions; + const ratingScale = update.ratingScale ?? existing?.ratingScale; + const modelId = + update.model ?? + (existing?.modelConfig && "bedrockEvaluatorModelConfig" in existing.modelConfig + ? existing.modelConfig.bedrockEvaluatorModelConfig?.modelId + : undefined); + + if (!instructions || !ratingScale || !modelId) { + throw new TypeError( + `Evaluator "${id}" is not an LLM-as-a-Judge evaluator or is missing configuration required to update it`, + ); + } + + const evaluatorConfig: EvaluatorConfig = { + llmAsAJudge: { + instructions, + ratingScale, + modelConfig: { bedrockEvaluatorModelConfig: { modelId } }, + }, + }; + + return control.send( + new UpdateEvaluatorCommand({ + evaluatorId: id, + evaluatorConfig, + kmsKeyArn: update.kmsKeyArn, + clientToken: update.clientToken, + }), + ); + } + + // updateCodeBasedEvaluator mirrors updateLlmAsAJudgeEvaluator: it merges the + // provided lambda ARN / timeout over the current codeBased config so unset + // fields are preserved across the union-replacing UpdateEvaluator call. + async updateCodeBasedEvaluator( + id: string, + update: CodeBasedUpdate, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); + const existing = + current.evaluatorConfig && "codeBased" in current.evaluatorConfig + ? current.evaluatorConfig.codeBased + : undefined; + const existingLambda = + existing && "lambdaConfig" in existing ? existing.lambdaConfig : undefined; + + const lambdaArn = update.lambdaArn ?? existingLambda?.lambdaArn; + if (!lambdaArn) { + throw new TypeError( + `Evaluator "${id}" is not a code-based evaluator or is missing configuration required to update it`, + ); + } + const lambdaTimeoutInSeconds = update.timeout ?? existingLambda?.lambdaTimeoutInSeconds; + + const evaluatorConfig: EvaluatorConfig = { + codeBased: { lambdaConfig: { lambdaArn, lambdaTimeoutInSeconds } }, + }; + + return control.send( + new UpdateEvaluatorCommand({ + evaluatorId: id, + evaluatorConfig, + kmsKeyArn: update.kmsKeyArn, + clientToken: update.clientToken, + }), + ); + } + + async getEvaluator(id: string, options: CoreOptions): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new GetEvaluatorCommand({ evaluatorId: id })); + } + + async listEvaluators( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new ListEvaluatorsCommand({ nextToken, maxResults })); + } + + async deleteEvaluator(id: string, options: CoreOptions): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new DeleteEvaluatorCommand({ evaluatorId: id })); + } +} diff --git a/src/core/index.tsx b/src/core/index.tsx index b08d01a49..702680512 100644 --- a/src/core/index.tsx +++ b/src/core/index.tsx @@ -1,6 +1,7 @@ import { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control"; import { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore"; import { IAMClient } from "@aws-sdk/client-iam"; +import { EvalClient } from "./eval"; import { HarnessClient } from "./harness"; import { IdentityClient } from "./identity"; import { RuntimeClient } from "./runtime"; @@ -48,6 +49,7 @@ export class CoreClient implements AwsClients { readonly harness: HarnessClient = new HarnessClient(this); readonly identity: IdentityClient = new IdentityClient(this); readonly runtime: RuntimeClient = new RuntimeClient(this); + readonly eval: EvalClient = new EvalClient(this); readonly projectManager: ProjectManager; diff --git a/src/handlers/eval/evaluator/code-based/index.tsx b/src/handlers/eval/evaluator/code-based/index.tsx new file mode 100644 index 000000000..362ab4662 --- /dev/null +++ b/src/handlers/eval/evaluator/code-based/index.tsx @@ -0,0 +1,89 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO, Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { SourceResolver } from "../../source"; + +const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; + +export const createCodeBasedCreateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create a code-based (Lambda-backed) evaluator", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), + flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), + // No default; the service applies its own timeout (60s) when omitted. + flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag( + "tags", + "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); + if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); + if (!flags["lambda-arn"]) { + throw new TypeError("required option '--lambda-arn ' not specified"); + } + + const source = new SourceResolver(io); + const tags = parseJsonFlag>( + "tags", + await source.resolve("tags", flags["tags"]), + ); + + const response = await core.eval.createEvaluator( + { + evaluatorName: flags["name"], + level: flags["level"], + evaluatorConfig: { + codeBased: { + lambdaConfig: { + lambdaArn: flags["lambda-arn"], + lambdaTimeoutInSeconds: flags["timeout"], + }, + }, + }, + kmsKeyArn: flags["kms-key-arn"], + tags, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); + +export const createCodeBasedUpdateHandler = (core: Core) => + createHandler({ + name: "update", + description: "update a code-based (Lambda-backed) evaluator", + flags: [ + flag("id", "the ID of the evaluator to update", z.string().optional()), + flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), + flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + + const response = await core.eval.updateCodeBasedEvaluator( + flags["id"], + { + lambdaArn: flags["lambda-arn"], + timeout: flags["timeout"], + kmsKeyArn: flags["kms-key-arn"], + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/delete/index.tsx b/src/handlers/eval/evaluator/delete/index.tsx new file mode 100644 index 000000000..9c2523985 --- /dev/null +++ b/src/handlers/eval/evaluator/delete/index.tsx @@ -0,0 +1,28 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteEvaluatorHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete an evaluator by id", + flags: [ + flag("id", "the ID of the evaluator to delete", z.string().optional()), + // This branch is headless/JSON only (no TUI prompt yet), so confirmation is + // required up front. `-y` shorthand is not wired: the router flag layer only + // supports long option names today. + flag("yes", "confirm deletion (required in non-interactive mode)", z.boolean().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + if (!flags["yes"]) { + throw new TypeError("refusing to delete without confirmation; pass '--yes' to proceed"); + } + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.deleteEvaluator(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx new file mode 100644 index 000000000..431d6a8cb --- /dev/null +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -0,0 +1,342 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { EvaluatorType, type EvaluatorSummary } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; +import { createRootHandler } from "../../index"; +import { ratingScaleFromPreset } from "../ratingScale"; + +const REGION = "us-west-2"; + +// run drives the real router (parsing → middleware → handler) against a +// TestCoreClient so we can assert on the exact request a handler built, plus the +// captured stdout. Optional `stdin` seeds the in-memory input stream for `-`. +function run(core: TestCoreClient, args: string[], stdin?: string) { + const io = testIO(); + if (stdin !== undefined) { + io.io.stdin.push(stdin); + io.io.stdin.push(null); + } + const root = createRootHandler(core, { io: io.io, logger: createSilentLogger() }); + return root.route(["node", "agentcore", ...args, "--region", REGION]).then(() => io.stdout()); +} + +describe("eval command hierarchy", () => { + test("registers the eval → evaluator command tree", () => { + const root = createRootHandler(new TestCoreClient(), { + io: testIO().io, + logger: createSilentLogger(), + }); + const evaluator = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "evaluator"); + + expect(evaluator?.children().map((c) => c.name())).toEqual([ + "llm-as-a-judge", + "code-based", + "get", + "list", + "delete", + ]); + expect( + evaluator + ?.children() + .find((c) => c.name() === "llm-as-a-judge") + ?.children() + .map((c) => c.name()), + ).toEqual(["create", "update"]); + expect( + evaluator + ?.children() + .find((c) => c.name() === "code-based") + ?.children() + .map((c) => c.name()), + ).toEqual(["create", "update"]); + }); + + test.each([ + "eval", + "eval evaluator", + "eval evaluator llm-as-a-judge", + "eval evaluator code-based", + ])("prints help for bare `%s` without an SDK call", async (command) => { + const core = new TestCoreClient(); + const stdout = await run(core, command.split(" ")); + expect(stdout).toContain(`Usage: agentcore ${command}`); + expect(core.eval.calls).toHaveLength(0); + }); +}); + +describe("llm-as-a-judge create", () => { + test("builds the request with a preset rating scale", async () => { + const core = new TestCoreClient(); + await run(core, [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "order-support-quality", + "--level", + "SESSION", + "--model", + "us.anthropic.claude-sonnet-4-5", + "--instructions", + "Judge the response.", + "--rating-scale", + "1-5-quality", + ]); + + expect(core.eval.calls).toHaveLength(1); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorName).toBe("order-support-quality"); + expect(request.level).toBe("SESSION"); + expect(request.evaluatorConfig.llmAsAJudge.instructions).toBe("Judge the response."); + expect( + request.evaluatorConfig.llmAsAJudge.modelConfig.bedrockEvaluatorModelConfig.modelId, + ).toBe("us.anthropic.claude-sonnet-4-5"); + expect(request.evaluatorConfig.llmAsAJudge.ratingScale).toEqual( + ratingScaleFromPreset("1-5-quality"), + ); + }); + + test("reads instructions from a file:// path", async () => { + const dir = mkdtempSync(join(tmpdir(), "eval-test-")); + const file = join(dir, "instructions.txt"); + writeFileSync(file, "Instructions from file."); + try { + const core = new TestCoreClient(); + await run(core, [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "x", + "--level", + "TRACE", + "--model", + "m", + "--instructions", + `file://${file}`, + "--rating-scale", + "pass-fail", + ]); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.llmAsAJudge.instructions).toBe("Instructions from file."); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("reads instructions from stdin with `-`", async () => { + const core = new TestCoreClient(); + await run( + core, + [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "x", + "--level", + "SESSION", + "--model", + "m", + "--instructions", + "-", + "--rating-scale", + "1-3-simple", + ], + "Instructions from stdin.", + ); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.llmAsAJudge.instructions).toBe("Instructions from stdin."); + }); + + test("accepts a custom rating scale via --rating-scale-json", async () => { + const core = new TestCoreClient(); + const scale = JSON.stringify({ numerical: [{ value: 1, label: "L", definition: "d" }] }); + await run(core, [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "x", + "--level", + "SESSION", + "--model", + "m", + "--instructions", + "i", + "--rating-scale-json", + scale, + ]); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.llmAsAJudge.ratingScale).toEqual(JSON.parse(scale)); + }); + + test.each([ + [ + "missing --name", + ["--level", "SESSION", "--model", "m", "--instructions", "i", "--rating-scale", "pass-fail"], + /--name/, + ], + [ + "missing --level", + ["--name", "x", "--model", "m", "--instructions", "i", "--rating-scale", "pass-fail"], + /--level/, + ], + [ + "missing --model", + ["--name", "x", "--level", "SESSION", "--instructions", "i", "--rating-scale", "pass-fail"], + /--model/, + ], + [ + "missing --instructions", + ["--name", "x", "--level", "SESSION", "--model", "m", "--rating-scale", "pass-fail"], + /--instructions/, + ], + [ + "missing rating scale", + ["--name", "x", "--level", "SESSION", "--model", "m", "--instructions", "i"], + /rating-scale/, + ], + ] as const)("rejects %s", async (_label, extra, message) => { + const core = new TestCoreClient(); + expect(run(core, ["eval", "evaluator", "llm-as-a-judge", "create", ...extra])).rejects.toThrow( + message, + ); + }); + + test("rejects both --rating-scale and --rating-scale-json", async () => { + const core = new TestCoreClient(); + expect( + run(core, [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "x", + "--level", + "SESSION", + "--model", + "m", + "--instructions", + "i", + "--rating-scale", + "pass-fail", + "--rating-scale-json", + "{}", + ]), + ).rejects.toThrow(/only one of/); + }); +}); + +describe("code-based create", () => { + test("builds the request and omits timeout when not given", async () => { + const core = new TestCoreClient(); + await run(core, [ + "eval", + "evaluator", + "code-based", + "create", + "--name", + "refund-policy", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:refund", + ]); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.codeBased.lambdaConfig.lambdaArn).toContain("function:refund"); + expect(request.evaluatorConfig.codeBased.lambdaConfig.lambdaTimeoutInSeconds).toBeUndefined(); + }); + + test("passes timeout through when given", async () => { + const core = new TestCoreClient(); + await run(core, [ + "eval", + "evaluator", + "code-based", + "create", + "--name", + "x", + "--level", + "SESSION", + "--lambda-arn", + "arn:x", + "--timeout", + "30", + ]); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.codeBased.lambdaConfig.lambdaTimeoutInSeconds).toBe(30); + }); + + test("rejects a missing --lambda-arn", async () => { + const core = new TestCoreClient(); + expect( + run(core, ["eval", "evaluator", "code-based", "create", "--name", "x", "--level", "SESSION"]), + ).rejects.toThrow(/--lambda-arn/); + }); +}); + +describe("update / get / delete required flags", () => { + test.each([ + ["llm-as-a-judge update", ["eval", "evaluator", "llm-as-a-judge", "update"]], + ["code-based update", ["eval", "evaluator", "code-based", "update"]], + ["get", ["eval", "evaluator", "get"]], + ["delete", ["eval", "evaluator", "delete"]], + ] as const)("`%s` requires --id", async (_label, args) => { + const core = new TestCoreClient(); + expect(run(core, [...args])).rejects.toThrow(/--id/); + }); + + test("delete requires --yes", async () => { + const core = new TestCoreClient(); + expect(run(core, ["eval", "evaluator", "delete", "--id", "e-1"])).rejects.toThrow(/--yes/); + }); + + test("delete proceeds with --yes", async () => { + const core = new TestCoreClient(); + await run(core, ["eval", "evaluator", "delete", "--id", "e-1", "--yes"]); + expect(core.eval.calls).toEqual([ + { method: "deleteEvaluator", args: ["e-1", { region: REGION, endpointUrl: undefined }] }, + ]); + }); +}); + +describe("list filtering", () => { + const evaluators = [ + { evaluatorId: "b1", evaluatorType: EvaluatorType.BUILTIN }, + { evaluatorId: "c1", evaluatorType: EvaluatorType.CODE }, + { evaluatorId: "l1", evaluatorType: EvaluatorType.CUSTOM }, + ] as unknown as EvaluatorSummary[]; + + test.each([ + ["Builtin", ["b1"]], + ["code-based", ["c1"]], + ["llm-as-a-judge", ["l1"]], + ] as const)("filters the returned page by --type %s", async (type, expectedIds) => { + const core = new TestCoreClient(); + core.eval.setListResponse({ evaluators }); + const stdout = await run(core, ["eval", "evaluator", "list", "--type", type, "--json"]); + const parsed = JSON.parse(stdout); + expect(parsed.evaluators.map((e: { evaluatorId: string }) => e.evaluatorId)).toEqual( + expectedIds, + ); + }); + + test("returns all evaluators when --type is omitted", async () => { + const core = new TestCoreClient(); + core.eval.setListResponse({ evaluators }); + const stdout = await run(core, ["eval", "evaluator", "list", "--json"]); + expect(JSON.parse(stdout).evaluators).toHaveLength(3); + }); +}); diff --git a/src/handlers/eval/evaluator/get/index.tsx b/src/handlers/eval/evaluator/get/index.tsx new file mode 100644 index 000000000..82185ef9c --- /dev/null +++ b/src/handlers/eval/evaluator/get/index.tsx @@ -0,0 +1,19 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetEvaluatorHandler = (core: Core) => + createHandler({ + name: "get", + description: "get an evaluator by id", + flags: [flag("id", "the ID of the evaluator", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.getEvaluator(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/evaluator/index.tsx b/src/handlers/eval/evaluator/index.tsx new file mode 100644 index 000000000..474aa4112 --- /dev/null +++ b/src/handlers/eval/evaluator/index.tsx @@ -0,0 +1,28 @@ +import { Router } from "../../../router"; +import type { AppIO, Core } from "../../types"; +import { createHelpDefault } from "../../help"; +import { createLlmAsAJudgeCreateHandler, createLlmAsAJudgeUpdateHandler } from "./llm-as-a-judge"; +import { createCodeBasedCreateHandler, createCodeBasedUpdateHandler } from "./code-based"; +import { createGetEvaluatorHandler } from "./get"; +import { createListEvaluatorsHandler } from "./list"; +import { createDeleteEvaluatorHandler } from "./delete"; + +export function createEvaluatorHandler(core: Core, io: AppIO): Router { + const llmAsAJudge = new Router("llm-as-a-judge", "manage LLM-as-a-Judge evaluators") + .default(createHelpDefault(io)) + .handler(createLlmAsAJudgeCreateHandler(core, io)) + .handler(createLlmAsAJudgeUpdateHandler(core, io)); + + const codeBased = new Router("code-based", "manage code-based (Lambda-backed) evaluators") + .default(createHelpDefault(io)) + .handler(createCodeBasedCreateHandler(core, io)) + .handler(createCodeBasedUpdateHandler(core)); + + return new Router("evaluator", "manage AgentCore evaluators") + .default(createHelpDefault(io)) + .handler(llmAsAJudge) + .handler(codeBased) + .handler(createGetEvaluatorHandler(core)) + .handler(createListEvaluatorsHandler(core)) + .handler(createDeleteEvaluatorHandler(core)); +} diff --git a/src/handlers/eval/evaluator/list/index.tsx b/src/handlers/eval/evaluator/list/index.tsx new file mode 100644 index 000000000..40680c59d --- /dev/null +++ b/src/handlers/eval/evaluator/list/index.tsx @@ -0,0 +1,47 @@ +import z from "zod"; +import { EvaluatorType } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +const TYPE_FILTERS = ["Builtin", "code-based", "llm-as-a-judge"] as const; + +// The AgentCore ListEvaluators API only paginates (nextToken/maxResults); it does +// not filter by type. `--type` is therefore applied client-side to the returned +// page, so combining it with --max-results can under-fill a page. +const TYPE_TO_SDK: Record<(typeof TYPE_FILTERS)[number], string> = { + Builtin: EvaluatorType.BUILTIN, + "code-based": EvaluatorType.CODE, + "llm-as-a-judge": EvaluatorType.CUSTOM, +}; + +export const createListEvaluatorsHandler = (core: Core) => + createHandler({ + name: "list", + description: "list evaluators", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + flag( + "type", + `filter the returned page by type (${TYPE_FILTERS.join(" | ")})`, + z.enum(TYPE_FILTERS).optional(), + ), + ], + handle: async (ctx, flags) => { + const response = await core.eval.listEvaluators( + flags["next-token"], + flags["max-results"], + coreOptsFromCtx(ctx), + ); + + const type = flags["type"]; + if (type && response.evaluators) { + const wanted = TYPE_TO_SDK[type]; + response.evaluators = response.evaluators.filter((e) => e.evaluatorType === wanted); + } + + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx new file mode 100644 index 000000000..797c067ce --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx @@ -0,0 +1,147 @@ +import z from "zod"; +import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO, Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { RATING_SCALE_PRESET_IDS, ratingScaleFromPreset } from "../../ratingScale"; +import { SourceResolver } from "../../source"; + +const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; + +// resolveRatingScale turns the mutually-exclusive --rating-scale / --rating-scale-json +// flags into a RatingScale, or undefined when neither is given. `source` resolves +// the JSON form's inline / file:// / stdin value before parsing. +async function resolveRatingScale( + preset: string | undefined, + json: string | undefined, + source: SourceResolver, +): Promise { + if (preset !== undefined && json !== undefined) { + throw new TypeError("pass only one of '--rating-scale' or '--rating-scale-json'"); + } + if (preset !== undefined) { + return ratingScaleFromPreset(preset as (typeof RATING_SCALE_PRESET_IDS)[number]); + } + const raw = await source.resolve("rating-scale-json", json); + return parseJsonFlag("rating-scale-json", raw); +} + +const ratingScaleFlags = [ + flag( + "rating-scale", + `rating scale preset (${RATING_SCALE_PRESET_IDS.join(" | ")})`, + z.enum(RATING_SCALE_PRESET_IDS).optional(), + ), + flag( + "rating-scale-json", + "custom rating scale (JSON RatingScale; inline, file://, or - for stdin)", + z.string().optional(), + ), +] as const; + +const instructionsFlag = flag( + "instructions", + "evaluation instructions (inline, file://, or - for stdin)", + z.string().optional(), +); + +export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create an LLM-as-a-Judge evaluator", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), + flag("model", "the Bedrock model id used to judge", z.string().optional()), + instructionsFlag, + ...ratingScaleFlags, + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag( + "tags", + "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); + if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); + if (!flags["model"]) throw new TypeError("required option '--model ' not specified"); + + const source = new SourceResolver(io); + const instructions = await source.resolve("instructions", flags["instructions"]); + if (!instructions) { + throw new TypeError("required option '--instructions ' not specified"); + } + const ratingScale = await resolveRatingScale( + flags["rating-scale"], + flags["rating-scale-json"], + source, + ); + if (!ratingScale) { + throw new TypeError("one of '--rating-scale' or '--rating-scale-json' is required"); + } + const tags = parseJsonFlag>( + "tags", + await source.resolve("tags", flags["tags"]), + ); + + const response = await core.eval.createEvaluator( + { + evaluatorName: flags["name"], + level: flags["level"], + evaluatorConfig: { + llmAsAJudge: { + instructions, + ratingScale, + modelConfig: { bedrockEvaluatorModelConfig: { modelId: flags["model"] } }, + }, + }, + kmsKeyArn: flags["kms-key-arn"], + tags, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); + +export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "update", + description: "update an LLM-as-a-Judge evaluator", + flags: [ + flag("id", "the ID of the evaluator to update", z.string().optional()), + instructionsFlag, + flag("model", "the Bedrock model id used to judge", z.string().optional()), + ...ratingScaleFlags, + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + + const source = new SourceResolver(io); + const instructions = await source.resolve("instructions", flags["instructions"]); + const ratingScale = await resolveRatingScale( + flags["rating-scale"], + flags["rating-scale-json"], + source, + ); + + const response = await core.eval.updateLlmAsAJudgeEvaluator( + flags["id"], + { + instructions, + model: flags["model"], + ratingScale, + kmsKeyArn: flags["kms-key-arn"], + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx new file mode 100644 index 000000000..668fe6e69 --- /dev/null +++ b/src/handlers/eval/index.tsx @@ -0,0 +1,10 @@ +import { Router } from "../../router"; +import type { AppIO, Core } from "../types"; +import { createHelpDefault } from "../help"; +import { createEvaluatorHandler } from "./evaluator"; + +export function createEvalHandler(core: Core, io: AppIO): Router { + return new Router("eval", "evaluate and optimize AgentCore agents") + .default(createHelpDefault(io)) + .handler(createEvaluatorHandler(core, io)); +} diff --git a/src/handlers/eval/ratingScale.tsx b/src/handlers/eval/ratingScale.tsx new file mode 100644 index 000000000..8752a1dd9 --- /dev/null +++ b/src/handlers/eval/ratingScale.tsx @@ -0,0 +1,50 @@ +import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; + +// Rating-scale presets. `--rating-scale ` expands to a full RatingScale +// union for the common cases; `--rating-scale-json` accepts a raw RatingScale for +// anything the presets don't cover (mirrors what the API supports directly). +export const RATING_SCALE_PRESET_IDS = [ + "1-5-quality", + "1-3-simple", + "pass-fail", + "good-neutral-bad", +] as const; + +export type RatingScalePresetId = (typeof RATING_SCALE_PRESET_IDS)[number]; + +const PRESETS: Record = { + "1-5-quality": { + numerical: [ + { value: 1, label: "Poor", definition: "Fails to meet expectations" }, + { value: 2, label: "Fair", definition: "Partially meets expectations" }, + { value: 3, label: "Good", definition: "Meets expectations" }, + { value: 4, label: "Very Good", definition: "Exceeds expectations" }, + { value: 5, label: "Excellent", definition: "Far exceeds expectations" }, + ], + }, + "1-3-simple": { + numerical: [ + { value: 1, label: "Low", definition: "Below acceptable quality" }, + { value: 2, label: "Medium", definition: "Acceptable quality" }, + { value: 3, label: "High", definition: "Above acceptable quality" }, + ], + }, + "pass-fail": { + categorical: [ + { label: "Pass", definition: "Meets the evaluation criteria" }, + { label: "Fail", definition: "Does not meet the evaluation criteria" }, + ], + }, + "good-neutral-bad": { + categorical: [ + { label: "Good", definition: "Positive outcome, meets or exceeds criteria" }, + { label: "Neutral", definition: "Acceptable but unremarkable outcome" }, + { label: "Bad", definition: "Negative outcome, fails to meet criteria" }, + ], + }, +}; + +// ratingScaleFromPreset returns the RatingScale for a preset id. +export function ratingScaleFromPreset(id: RatingScalePresetId): RatingScale { + return PRESETS[id]; +} diff --git a/src/handlers/eval/source.test.ts b/src/handlers/eval/source.test.ts new file mode 100644 index 000000000..aa0ae452e --- /dev/null +++ b/src/handlers/eval/source.test.ts @@ -0,0 +1,52 @@ +import { test, expect } from "bun:test"; +import { PassThrough } from "node:stream"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AppIO } from "../types"; +import { SourceResolver } from "./source"; + +function ioWithStdin(input: string): AppIO { + const stdin = new PassThrough() as unknown as NodeJS.ReadStream; + stdin.push(input); + stdin.push(null); + return { + stdin, + stdout: new PassThrough() as unknown as NodeJS.WriteStream, + stderr: new PassThrough() as unknown as NodeJS.WriteStream, + }; +} + +test("resolve returns inline values and passes undefined through", async () => { + const r = new SourceResolver(ioWithStdin("")); + expect(await r.resolve("f", "hello")).toBe("hello"); + expect(await r.resolve("f", undefined)).toBeUndefined(); +}); + +test("resolve reads file:// paths", async () => { + const dir = mkdtempSync(join(tmpdir(), "source-test-")); + const file = join(dir, "v.txt"); + writeFileSync(file, "from file"); + try { + const r = new SourceResolver(ioWithStdin("")); + expect(await r.resolve("f", `file://${file}`)).toBe("from file"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("resolve reads stdin for `-`", async () => { + const r = new SourceResolver(ioWithStdin("from stdin")); + expect(await r.resolve("f", "-")).toBe("from stdin"); +}); + +test("resolve rejects a second stdin source", async () => { + const r = new SourceResolver(ioWithStdin("only once")); + await r.resolve("a", "-"); + await expect(r.resolve("b", "-")).rejects.toThrow(/only one option may read from stdin/); +}); + +test("resolve reports a helpful error for a missing file", async () => { + const r = new SourceResolver(ioWithStdin("")); + await expect(r.resolve("f", "file:///nope/missing.txt")).rejects.toThrow(/could not read/); +}); diff --git a/src/handlers/eval/source.tsx b/src/handlers/eval/source.tsx new file mode 100644 index 000000000..45214a42a --- /dev/null +++ b/src/handlers/eval/source.tsx @@ -0,0 +1,58 @@ +import type { AppIO } from "../types"; + +// A field value can be supplied inline, read from a file (`file://`), or +// read from stdin (`-`), following the AWS CLI `file://` convention. This avoids +// a companion `--*-file` flag for every eligible field. Each occurrence selects +// exactly one source, and a command accepts at most one stdin source. + +const FILE_PREFIX = "file://"; +const STDIN = "-"; + +// stdinReader tracks whether stdin has already been claimed within a single +// command invocation, so a second `-` is rejected rather than silently reading +// an exhausted stream. +export class SourceResolver { + private stdinClaimed = false; + + constructor(private readonly io: AppIO) {} + + // resolve returns the effective value for a source-aware flag: the raw string + // inline, the contents of `file://`, or stdin for `-`. Undefined passes + // through so optional flags stay omitted. + async resolve(name: string, raw: string | undefined): Promise { + if (raw === undefined) return undefined; + + if (raw === STDIN) { + if (this.stdinClaimed) { + throw new TypeError( + `only one option may read from stdin ('-'); '--${name}' cannot also read stdin`, + ); + } + this.stdinClaimed = true; + return readStream(this.io.stdin); + } + + if (raw.startsWith(FILE_PREFIX)) { + const path = raw.slice(FILE_PREFIX.length); + try { + return await Bun.file(path).text(); + } catch (error) { + throw new TypeError( + `could not read '--${name}' from file '${path}': ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + } + + return raw; + } +} + +async function readStream(stream: NodeJS.ReadStream): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks).toString("utf8"); +} diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx new file mode 100644 index 000000000..20308424c --- /dev/null +++ b/src/handlers/eval/types.tsx @@ -0,0 +1,62 @@ +import type { + CreateEvaluatorRequest, + CreateEvaluatorResponse, + DeleteEvaluatorResponse, + GetEvaluatorResponse, + ListEvaluatorsResponse, + RatingScale, + UpdateEvaluatorResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import type { CoreOptions } from "../../core/types"; + +// LlmAsAJudgeUpdate carries the fields a caller may change on an LLM-as-a-Judge +// evaluator. Any field left undefined is preserved from the existing evaluator: +// the AgentCore UpdateEvaluator API replaces the whole evaluatorConfig union, and +// the llmAsAJudge arm requires instructions + ratingScale + modelConfig together, +// so a partial update is only possible by merging over the current definition. +export interface LlmAsAJudgeUpdate { + instructions?: string; + model?: string; + ratingScale?: RatingScale; + kmsKeyArn?: string; + clientToken?: string; +} + +// CodeBasedUpdate carries the fields a caller may change on a code-based +// evaluator. Undefined fields are preserved from the existing evaluator, for the +// same union-replacement reason described on LlmAsAJudgeUpdate. +export interface CodeBasedUpdate { + lambdaArn?: string; + timeout?: number; + kmsKeyArn?: string; + clientToken?: string; +} + +// CoreEvalClient is the evaluator surface the eval handlers depend on. It is +// declared here, next to the handlers that consume it, and implemented by +// src/core/eval.tsx (dependency inversion: handlers own the abstraction). +export interface CoreEvalClient { + createEvaluator( + request: CreateEvaluatorRequest, + options: CoreOptions, + ): Promise; + // update*Evaluator fetch the current evaluator and merge the provided fields + // before sending, because the API replaces the entire evaluatorConfig union. + updateLlmAsAJudgeEvaluator( + id: string, + update: LlmAsAJudgeUpdate, + options: CoreOptions, + ): Promise; + updateCodeBasedEvaluator( + id: string, + update: CodeBasedUpdate, + options: CoreOptions, + ): Promise; + getEvaluator(id: string, options: CoreOptions): Promise; + listEvaluators( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + deleteEvaluator(id: string, options: CoreOptions): Promise; +} diff --git a/src/handlers/index.tsx b/src/handlers/index.tsx index 047e2f619..0b1e03fff 100644 --- a/src/handlers/index.tsx +++ b/src/handlers/index.tsx @@ -1,4 +1,5 @@ import { Router } from "../router"; +import { createEvalHandler } from "./eval/index.tsx"; import { createHarnessHandler } from "./harness/index.tsx"; import { createIdentityHandler } from "./identity/index.tsx"; import { createRuntimeHandler } from "./runtime/index.tsx"; @@ -37,6 +38,7 @@ export function createRootHandler(core: Core, config: RootHandlerConfig): Router root.handler(createHarnessHandler(core, io)); root.handler(createIdentityHandler(core, io)); root.handler(createRuntimeHandler(core, io)); + root.handler(createEvalHandler(core, io)); root.handler(createConfigHandler(io)); root.handler(createProjectHandler({ projectManager: core.projectManager })); diff --git a/src/handlers/root.test.tsx b/src/handlers/root.test.tsx index e217ef582..a9cbec735 100644 --- a/src/handlers/root.test.tsx +++ b/src/handlers/root.test.tsx @@ -13,6 +13,7 @@ describe("createRootHandler", () => { "harness", "identity", "runtime", + "eval", "config", "project", ]); diff --git a/src/handlers/types.tsx b/src/handlers/types.tsx index 8f8371d7a..959ddc3d7 100644 --- a/src/handlers/types.tsx +++ b/src/handlers/types.tsx @@ -1,3 +1,4 @@ +import type { CoreEvalClient } from "./eval/types.tsx"; import type { CoreHarnessClient } from "./harness/types.tsx"; import type { CoreIdentityClient } from "./identity/types.tsx"; import type { CoreRuntimeClient } from "./runtime/types.tsx"; @@ -8,6 +9,7 @@ export interface Core { harness: CoreHarnessClient; identity: CoreIdentityClient; runtime: CoreRuntimeClient; + eval: CoreEvalClient; projectManager: ProjectManager; } diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 531c2df07..8411ee99b 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -20,6 +20,12 @@ import type { ListHarnessesResponse, ListHarnessEndpointsResponse, ListHarnessVersionsResponse, + CreateEvaluatorRequest, + CreateEvaluatorResponse, + DeleteEvaluatorResponse, + GetEvaluatorResponse, + ListEvaluatorsResponse, + UpdateEvaluatorResponse, UpdateApiKeyCredentialProviderResponse, UpdateHarnessEndpointRequest, UpdateHarnessEndpointResponse, @@ -38,6 +44,7 @@ import type { Core } from "../handlers/types"; import type { CoreHarnessClient, CreateHarnessInput } from "../handlers/harness/types"; import type { CoreIdentityClient } from "../handlers/identity/types"; import type { CoreRuntimeClient } from "../handlers/runtime/types"; +import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; import type { CoreOptions } from "../core/types"; import type { ProjectManager } from "../handlers/project/types"; import type { Logger } from "../logging"; @@ -96,6 +103,11 @@ const DEFAULT_LIST_RUNTIME_VERSIONS_RESPONSE: ListAgentRuntimeVersionsResponse = const DEFAULT_LIST_RUNTIME_ENDPOINTS_RESPONSE: ListAgentRuntimeEndpointsResponse = { runtimeEndpoints: [], }; +const DEFAULT_CREATE_EVALUATOR_RESPONSE = {} as CreateEvaluatorResponse; +const DEFAULT_UPDATE_EVALUATOR_RESPONSE = {} as UpdateEvaluatorResponse; +const DEFAULT_GET_EVALUATOR_RESPONSE = {} as GetEvaluatorResponse; +const DEFAULT_LIST_EVALUATORS_RESPONSE: ListEvaluatorsResponse = { evaluators: [] }; +const DEFAULT_DELETE_EVALUATOR_RESPONSE = {} as DeleteEvaluatorResponse; // abortError mirrors the error the SDK's abort handling rejects with. function abortError(): Error { @@ -606,11 +618,86 @@ class TestIdentityClient implements CoreIdentityClient { } } +// TestEvalClient records every call and returns canned responses. Configure the +// list response to exercise the client-side `--type` filter, or set an error to +// make the next calls throw. +export class TestEvalClient implements CoreEvalClient { + readonly calls: RecordedCall[] = []; + private listResponse: ListEvaluatorsResponse = DEFAULT_LIST_EVALUATORS_RESPONSE; + private error?: Error; + + setListResponse(response: ListEvaluatorsResponse): void { + this.listResponse = response; + } + + setError(error: Error): void { + this.error = error; + } + + private maybeThrow(): void { + if (this.error) throw this.error; + } + + async createEvaluator( + request: CreateEvaluatorRequest, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "createEvaluator", args: [request, options] }); + this.maybeThrow(); + return DEFAULT_CREATE_EVALUATOR_RESPONSE; + } + + async updateLlmAsAJudgeEvaluator( + id: string, + update: LlmAsAJudgeUpdate, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "updateLlmAsAJudgeEvaluator", args: [id, update, options] }); + this.maybeThrow(); + return DEFAULT_UPDATE_EVALUATOR_RESPONSE; + } + + async updateCodeBasedEvaluator( + id: string, + update: CodeBasedUpdate, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "updateCodeBasedEvaluator", args: [id, update, options] }); + this.maybeThrow(); + return DEFAULT_UPDATE_EVALUATOR_RESPONSE; + } + + async getEvaluator(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "getEvaluator", args: [id, options] }); + this.maybeThrow(); + return DEFAULT_GET_EVALUATOR_RESPONSE; + } + + async listEvaluators( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "listEvaluators", args: [nextToken, maxResults, options] }); + this.maybeThrow(); + // Return a fresh copy so a handler's client-side filtering can't mutate the + // configured response between calls. + return { ...this.listResponse, evaluators: [...(this.listResponse.evaluators ?? [])] }; + } + + async deleteEvaluator(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "deleteEvaluator", args: [id, options] }); + this.maybeThrow(); + return DEFAULT_DELETE_EVALUATOR_RESPONSE; + } +} + // TestCoreClient implements the Core contract with fully controllable sub-clients. export class TestCoreClient implements Core { readonly harness = new TestHarnessClient(); readonly identity = new TestIdentityClient(); readonly runtime = new TestRuntimeClient(); + readonly eval = new TestEvalClient(); readonly projectManager: ProjectManager; constructor(options?: TestCoreClientOptions) { diff --git a/src/testing/index.tsx b/src/testing/index.tsx index e3e711ee6..eec7903cc 100644 --- a/src/testing/index.tsx +++ b/src/testing/index.tsx @@ -6,6 +6,7 @@ export { TestCoreClient, TestHarnessClient, TestRuntimeClient, + TestEvalClient, type RecordedCall, } from "./TestCoreClient"; export { StreamController } from "./StreamController"; From f90d73604be21a1ac7817390dc0eb5f68eda0ad0 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 23 Jul 2026 22:19:37 +0000 Subject: [PATCH 2/4] fix(eval): lowercase 'builtin' in list --type filter --- README.md | 2 +- src/handlers/eval/evaluator/evaluator.test.tsx | 2 +- src/handlers/eval/evaluator/list/index.tsx | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 33a78604e..87c76c514 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ agentcore eval evaluator code-based create \ --lambda-arn arn:aws:lambda:us-west-2:123456789012:function:refund-policy \ --json -# Get, list, delete. --type filters the returned page (Builtin | code-based | llm-as-a-judge). +# Get, list, delete. --type filters the returned page (builtin | code-based | llm-as-a-judge). agentcore eval evaluator get --id --json agentcore eval evaluator list --type llm-as-a-judge --max-results 20 --json agentcore eval evaluator delete --id --yes --json diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index 431d6a8cb..519e5c38d 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -320,7 +320,7 @@ describe("list filtering", () => { ] as unknown as EvaluatorSummary[]; test.each([ - ["Builtin", ["b1"]], + ["builtin", ["b1"]], ["code-based", ["c1"]], ["llm-as-a-judge", ["l1"]], ] as const)("filters the returned page by --type %s", async (type, expectedIds) => { diff --git a/src/handlers/eval/evaluator/list/index.tsx b/src/handlers/eval/evaluator/list/index.tsx index 40680c59d..581430672 100644 --- a/src/handlers/eval/evaluator/list/index.tsx +++ b/src/handlers/eval/evaluator/list/index.tsx @@ -5,13 +5,13 @@ import { JsonRendererKey } from "../../../../tui"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; -const TYPE_FILTERS = ["Builtin", "code-based", "llm-as-a-judge"] as const; +const TYPE_FILTERS = ["builtin", "code-based", "llm-as-a-judge"] as const; // The AgentCore ListEvaluators API only paginates (nextToken/maxResults); it does // not filter by type. `--type` is therefore applied client-side to the returned // page, so combining it with --max-results can under-fill a page. const TYPE_TO_SDK: Record<(typeof TYPE_FILTERS)[number], string> = { - Builtin: EvaluatorType.BUILTIN, + builtin: EvaluatorType.BUILTIN, "code-based": EvaluatorType.CODE, "llm-as-a-judge": EvaluatorType.CUSTOM, }; From 02e4def53542c837a55baa5bae5f2efdcd8141a5 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 23 Jul 2026 22:25:13 +0000 Subject: [PATCH 3/4] refactor(eval): merge --rating-scale-json into --rating-scale A single --rating-scale flag now takes either a preset id or a source-aware custom RatingScale (JSON inline, file://, or -). A value matching a known preset id expands to that preset; anything else is parsed as a RatingScale JSON value. --- .../eval/evaluator/evaluator.test.tsx | 42 ++++++++++-- .../eval/evaluator/llm-as-a-judge/index.tsx | 64 ++++++++----------- src/handlers/eval/ratingScale.tsx | 11 +++- 3 files changed, 69 insertions(+), 48 deletions(-) diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index 519e5c38d..878a81215 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -158,7 +158,7 @@ describe("llm-as-a-judge create", () => { expect(request.evaluatorConfig.llmAsAJudge.instructions).toBe("Instructions from stdin."); }); - test("accepts a custom rating scale via --rating-scale-json", async () => { + test("accepts a custom rating scale as inline JSON on --rating-scale", async () => { const core = new TestCoreClient(); const scale = JSON.stringify({ numerical: [{ value: 1, label: "L", definition: "d" }] }); await run(core, [ @@ -174,13 +174,43 @@ describe("llm-as-a-judge create", () => { "m", "--instructions", "i", - "--rating-scale-json", + "--rating-scale", scale, ]); const [request] = core.eval.calls[0]!.args as [any]; expect(request.evaluatorConfig.llmAsAJudge.ratingScale).toEqual(JSON.parse(scale)); }); + test("reads a custom rating scale from a file:// path on --rating-scale", async () => { + const dir = mkdtempSync(join(tmpdir(), "eval-test-")); + const file = join(dir, "scale.json"); + const scale = { categorical: [{ label: "P", definition: "pass" }] }; + writeFileSync(file, JSON.stringify(scale)); + try { + const core = new TestCoreClient(); + await run(core, [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "x", + "--level", + "SESSION", + "--model", + "m", + "--instructions", + "i", + "--rating-scale", + `file://${file}`, + ]); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.llmAsAJudge.ratingScale).toEqual(scale); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + test.each([ [ "missing --name", @@ -214,7 +244,7 @@ describe("llm-as-a-judge create", () => { ); }); - test("rejects both --rating-scale and --rating-scale-json", async () => { + test("rejects malformed custom rating scale JSON", async () => { const core = new TestCoreClient(); expect( run(core, [ @@ -231,11 +261,9 @@ describe("llm-as-a-judge create", () => { "--instructions", "i", "--rating-scale", - "pass-fail", - "--rating-scale-json", - "{}", + "{not json", ]), - ).rejects.toThrow(/only one of/); + ).rejects.toThrow(/Invalid JSON for option '--rating-scale'/); }); }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx index 797c067ce..27dbd0034 100644 --- a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx +++ b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx @@ -4,41 +4,35 @@ import { createHandler, flag } from "../../../../router"; import { JsonRendererKey } from "../../../../tui"; import type { AppIO, Core } from "../../../types"; import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; -import { RATING_SCALE_PRESET_IDS, ratingScaleFromPreset } from "../../ratingScale"; +import { + RATING_SCALE_PRESET_IDS, + isRatingScalePreset, + ratingScaleFromPreset, +} from "../../ratingScale"; import { SourceResolver } from "../../source"; const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; -// resolveRatingScale turns the mutually-exclusive --rating-scale / --rating-scale-json -// flags into a RatingScale, or undefined when neither is given. `source` resolves -// the JSON form's inline / file:// / stdin value before parsing. +// resolveRatingScale turns the single --rating-scale value into a RatingScale, or +// undefined when the flag is omitted. A value matching a known preset id expands +// to that preset; anything else is a source-aware JSON RatingScale (inline, +// file://, or - for stdin). A file literally named after a preset is still +// reachable via file://. async function resolveRatingScale( - preset: string | undefined, - json: string | undefined, + value: string | undefined, source: SourceResolver, ): Promise { - if (preset !== undefined && json !== undefined) { - throw new TypeError("pass only one of '--rating-scale' or '--rating-scale-json'"); - } - if (preset !== undefined) { - return ratingScaleFromPreset(preset as (typeof RATING_SCALE_PRESET_IDS)[number]); - } - const raw = await source.resolve("rating-scale-json", json); - return parseJsonFlag("rating-scale-json", raw); + if (value === undefined) return undefined; + if (isRatingScalePreset(value)) return ratingScaleFromPreset(value); + const raw = await source.resolve("rating-scale", value); + return parseJsonFlag("rating-scale", raw); } -const ratingScaleFlags = [ - flag( - "rating-scale", - `rating scale preset (${RATING_SCALE_PRESET_IDS.join(" | ")})`, - z.enum(RATING_SCALE_PRESET_IDS).optional(), - ), - flag( - "rating-scale-json", - "custom rating scale (JSON RatingScale; inline, file://, or - for stdin)", - z.string().optional(), - ), -] as const; +const ratingScaleFlag = flag( + "rating-scale", + `rating scale: a preset (${RATING_SCALE_PRESET_IDS.join(" | ")}) or a custom RatingScale (JSON inline, file://, or - for stdin)`, + z.string().optional(), +); const instructionsFlag = flag( "instructions", @@ -55,7 +49,7 @@ export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), flag("model", "the Bedrock model id used to judge", z.string().optional()), instructionsFlag, - ...ratingScaleFlags, + ratingScaleFlag, flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), flag( "tags", @@ -74,13 +68,9 @@ export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => if (!instructions) { throw new TypeError("required option '--instructions ' not specified"); } - const ratingScale = await resolveRatingScale( - flags["rating-scale"], - flags["rating-scale-json"], - source, - ); + const ratingScale = await resolveRatingScale(flags["rating-scale"], source); if (!ratingScale) { - throw new TypeError("one of '--rating-scale' or '--rating-scale-json' is required"); + throw new TypeError("required option '--rating-scale ' not specified"); } const tags = parseJsonFlag>( "tags", @@ -116,7 +106,7 @@ export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => flag("id", "the ID of the evaluator to update", z.string().optional()), instructionsFlag, flag("model", "the Bedrock model id used to judge", z.string().optional()), - ...ratingScaleFlags, + ratingScaleFlag, flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), flag("client-token", "idempotency token", z.string().optional()), ], @@ -125,11 +115,7 @@ export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => const source = new SourceResolver(io); const instructions = await source.resolve("instructions", flags["instructions"]); - const ratingScale = await resolveRatingScale( - flags["rating-scale"], - flags["rating-scale-json"], - source, - ); + const ratingScale = await resolveRatingScale(flags["rating-scale"], source); const response = await core.eval.updateLlmAsAJudgeEvaluator( flags["id"], diff --git a/src/handlers/eval/ratingScale.tsx b/src/handlers/eval/ratingScale.tsx index 8752a1dd9..3d40d7931 100644 --- a/src/handlers/eval/ratingScale.tsx +++ b/src/handlers/eval/ratingScale.tsx @@ -1,8 +1,9 @@ import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; // Rating-scale presets. `--rating-scale ` expands to a full RatingScale -// union for the common cases; `--rating-scale-json` accepts a raw RatingScale for -// anything the presets don't cover (mirrors what the API supports directly). +// union for the common cases; the same flag also accepts a raw RatingScale (JSON, +// inline / file:// / stdin) for anything the presets don't cover, mirroring what +// the API supports directly. export const RATING_SCALE_PRESET_IDS = [ "1-5-quality", "1-3-simple", @@ -48,3 +49,9 @@ const PRESETS: Record = { export function ratingScaleFromPreset(id: RatingScalePresetId): RatingScale { return PRESETS[id]; } + +// isRatingScalePreset reports whether a raw --rating-scale value names a preset. +// Anything else is treated as a source-aware JSON value (inline / file:// / -). +export function isRatingScalePreset(value: string): value is RatingScalePresetId { + return (RATING_SCALE_PRESET_IDS as readonly string[]).includes(value); +} From 6db704dab8c8c71a455cbc2cbc929e850b85eaba Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 24 Jul 2026 17:39:05 +0000 Subject: [PATCH 4/4] =?UTF-8?q?refactor(eval):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20per-subcommand=20dirs,=20type=20aliases,=20drop=20-?= =?UTF-8?q?-yes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Split llm-as-a-judge and code-based create/update into their own directories per the documented handler convention; shared flags/helpers live in a sibling utils.tsx. - LlmAsAJudgeUpdate / CodeBasedUpdate are now type aliases (data-carrying), leaving CoreEvalClient as the implemented interface. - Drop --yes from evaluator delete to stay consistent with the other CRUDL commands. --- .../evaluator/code-based/create/index.tsx | 61 ++++++++ .../eval/evaluator/code-based/index.tsx | 99 ++---------- .../evaluator/code-based/update/index.tsx | 33 ++++ src/handlers/eval/evaluator/delete/index.tsx | 11 +- .../eval/evaluator/evaluator.test.tsx | 9 +- src/handlers/eval/evaluator/index.tsx | 18 +-- .../evaluator/llm-as-a-judge/create/index.tsx | 65 ++++++++ .../eval/evaluator/llm-as-a-judge/index.tsx | 141 ++---------------- .../evaluator/llm-as-a-judge/update/index.tsx | 41 +++++ .../eval/evaluator/llm-as-a-judge/utils.tsx | 39 +++++ src/handlers/eval/types.tsx | 8 +- 11 files changed, 271 insertions(+), 254 deletions(-) create mode 100644 src/handlers/eval/evaluator/code-based/create/index.tsx create mode 100644 src/handlers/eval/evaluator/code-based/update/index.tsx create mode 100644 src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx create mode 100644 src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx create mode 100644 src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx diff --git a/src/handlers/eval/evaluator/code-based/create/index.tsx b/src/handlers/eval/evaluator/code-based/create/index.tsx new file mode 100644 index 000000000..2fe19e643 --- /dev/null +++ b/src/handlers/eval/evaluator/code-based/create/index.tsx @@ -0,0 +1,61 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { JsonRendererKey } from "../../../../../tui"; +import type { AppIO, Core } from "../../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; +import { SourceResolver } from "../../../source"; + +const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; + +export const createCodeBasedCreateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create a code-based (Lambda-backed) evaluator", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), + flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), + // No default; the service applies its own timeout (60s) when omitted. + flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag( + "tags", + "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); + if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); + if (!flags["lambda-arn"]) { + throw new TypeError("required option '--lambda-arn ' not specified"); + } + + const source = new SourceResolver(io); + const tags = parseJsonFlag>( + "tags", + await source.resolve("tags", flags["tags"]), + ); + + const response = await core.eval.createEvaluator( + { + evaluatorName: flags["name"], + level: flags["level"], + evaluatorConfig: { + codeBased: { + lambdaConfig: { + lambdaArn: flags["lambda-arn"], + lambdaTimeoutInSeconds: flags["timeout"], + }, + }, + }, + kmsKeyArn: flags["kms-key-arn"], + tags, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/code-based/index.tsx b/src/handlers/eval/evaluator/code-based/index.tsx index 362ab4662..14949f2ac 100644 --- a/src/handlers/eval/evaluator/code-based/index.tsx +++ b/src/handlers/eval/evaluator/code-based/index.tsx @@ -1,89 +1,12 @@ -import z from "zod"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; +import { Router } from "../../../../router"; import type { AppIO, Core } from "../../../types"; -import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; -import { SourceResolver } from "../../source"; - -const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; - -export const createCodeBasedCreateHandler = (core: Core, io: AppIO) => - createHandler({ - name: "create", - description: "create a code-based (Lambda-backed) evaluator", - flags: [ - flag("name", "the name of the evaluator", z.string().optional()), - flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), - flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), - // No default; the service applies its own timeout (60s) when omitted. - flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), - flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), - flag( - "tags", - "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", - z.string().optional(), - ), - flag("client-token", "idempotency token", z.string().optional()), - ], - handle: async (ctx, flags) => { - if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); - if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); - if (!flags["lambda-arn"]) { - throw new TypeError("required option '--lambda-arn ' not specified"); - } - - const source = new SourceResolver(io); - const tags = parseJsonFlag>( - "tags", - await source.resolve("tags", flags["tags"]), - ); - - const response = await core.eval.createEvaluator( - { - evaluatorName: flags["name"], - level: flags["level"], - evaluatorConfig: { - codeBased: { - lambdaConfig: { - lambdaArn: flags["lambda-arn"], - lambdaTimeoutInSeconds: flags["timeout"], - }, - }, - }, - kmsKeyArn: flags["kms-key-arn"], - tags, - clientToken: flags["client-token"], - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); - -export const createCodeBasedUpdateHandler = (core: Core) => - createHandler({ - name: "update", - description: "update a code-based (Lambda-backed) evaluator", - flags: [ - flag("id", "the ID of the evaluator to update", z.string().optional()), - flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), - flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), - flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), - flag("client-token", "idempotency token", z.string().optional()), - ], - handle: async (ctx, flags) => { - if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); - - const response = await core.eval.updateCodeBasedEvaluator( - flags["id"], - { - lambdaArn: flags["lambda-arn"], - timeout: flags["timeout"], - kmsKeyArn: flags["kms-key-arn"], - clientToken: flags["client-token"], - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); +import { createHelpDefault } from "../../../help"; +import { createCodeBasedCreateHandler } from "./create"; +import { createCodeBasedUpdateHandler } from "./update"; + +export function createCodeBasedHandler(core: Core, io: AppIO): Router { + return new Router("code-based", "manage code-based (Lambda-backed) evaluators") + .default(createHelpDefault(io)) + .handler(createCodeBasedCreateHandler(core, io)) + .handler(createCodeBasedUpdateHandler(core)); +} diff --git a/src/handlers/eval/evaluator/code-based/update/index.tsx b/src/handlers/eval/evaluator/code-based/update/index.tsx new file mode 100644 index 000000000..db5139a3c --- /dev/null +++ b/src/handlers/eval/evaluator/code-based/update/index.tsx @@ -0,0 +1,33 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { JsonRendererKey } from "../../../../../tui"; +import type { Core } from "../../../../types"; +import { coreOptsFromCtx } from "../../../../utils"; + +export const createCodeBasedUpdateHandler = (core: Core) => + createHandler({ + name: "update", + description: "update a code-based (Lambda-backed) evaluator", + flags: [ + flag("id", "the ID of the evaluator to update", z.string().optional()), + flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), + flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + + const response = await core.eval.updateCodeBasedEvaluator( + flags["id"], + { + lambdaArn: flags["lambda-arn"], + timeout: flags["timeout"], + kmsKeyArn: flags["kms-key-arn"], + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/delete/index.tsx b/src/handlers/eval/evaluator/delete/index.tsx index 9c2523985..78fe274a2 100644 --- a/src/handlers/eval/evaluator/delete/index.tsx +++ b/src/handlers/eval/evaluator/delete/index.tsx @@ -8,18 +8,9 @@ export const createDeleteEvaluatorHandler = (core: Core) => createHandler({ name: "delete", description: "delete an evaluator by id", - flags: [ - flag("id", "the ID of the evaluator to delete", z.string().optional()), - // This branch is headless/JSON only (no TUI prompt yet), so confirmation is - // required up front. `-y` shorthand is not wired: the router flag layer only - // supports long option names today. - flag("yes", "confirm deletion (required in non-interactive mode)", z.boolean().optional()), - ], + flags: [flag("id", "the ID of the evaluator to delete", z.string().optional())], handle: async (ctx, flags) => { if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); - if (!flags["yes"]) { - throw new TypeError("refusing to delete without confirmation; pass '--yes' to proceed"); - } ctx .require(JsonRendererKey) diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index 878a81215..4ff78135e 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -326,14 +326,9 @@ describe("update / get / delete required flags", () => { expect(run(core, [...args])).rejects.toThrow(/--id/); }); - test("delete requires --yes", async () => { + test("delete calls deleteEvaluator with the id", async () => { const core = new TestCoreClient(); - expect(run(core, ["eval", "evaluator", "delete", "--id", "e-1"])).rejects.toThrow(/--yes/); - }); - - test("delete proceeds with --yes", async () => { - const core = new TestCoreClient(); - await run(core, ["eval", "evaluator", "delete", "--id", "e-1", "--yes"]); + await run(core, ["eval", "evaluator", "delete", "--id", "e-1"]); expect(core.eval.calls).toEqual([ { method: "deleteEvaluator", args: ["e-1", { region: REGION, endpointUrl: undefined }] }, ]); diff --git a/src/handlers/eval/evaluator/index.tsx b/src/handlers/eval/evaluator/index.tsx index 474aa4112..e61a1bf68 100644 --- a/src/handlers/eval/evaluator/index.tsx +++ b/src/handlers/eval/evaluator/index.tsx @@ -1,27 +1,17 @@ import { Router } from "../../../router"; import type { AppIO, Core } from "../../types"; import { createHelpDefault } from "../../help"; -import { createLlmAsAJudgeCreateHandler, createLlmAsAJudgeUpdateHandler } from "./llm-as-a-judge"; -import { createCodeBasedCreateHandler, createCodeBasedUpdateHandler } from "./code-based"; +import { createLlmAsAJudgeHandler } from "./llm-as-a-judge"; +import { createCodeBasedHandler } from "./code-based"; import { createGetEvaluatorHandler } from "./get"; import { createListEvaluatorsHandler } from "./list"; import { createDeleteEvaluatorHandler } from "./delete"; export function createEvaluatorHandler(core: Core, io: AppIO): Router { - const llmAsAJudge = new Router("llm-as-a-judge", "manage LLM-as-a-Judge evaluators") - .default(createHelpDefault(io)) - .handler(createLlmAsAJudgeCreateHandler(core, io)) - .handler(createLlmAsAJudgeUpdateHandler(core, io)); - - const codeBased = new Router("code-based", "manage code-based (Lambda-backed) evaluators") - .default(createHelpDefault(io)) - .handler(createCodeBasedCreateHandler(core, io)) - .handler(createCodeBasedUpdateHandler(core)); - return new Router("evaluator", "manage AgentCore evaluators") .default(createHelpDefault(io)) - .handler(llmAsAJudge) - .handler(codeBased) + .handler(createLlmAsAJudgeHandler(core, io)) + .handler(createCodeBasedHandler(core, io)) .handler(createGetEvaluatorHandler(core)) .handler(createListEvaluatorsHandler(core)) .handler(createDeleteEvaluatorHandler(core)); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx new file mode 100644 index 000000000..33e0f7a95 --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx @@ -0,0 +1,65 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { JsonRendererKey } from "../../../../../tui"; +import type { AppIO, Core } from "../../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; +import { SourceResolver } from "../../../source"; +import { LEVELS, instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../utils"; + +export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create an LLM-as-a-Judge evaluator", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), + flag("model", "the Bedrock model id used to judge", z.string().optional()), + instructionsFlag, + ratingScaleFlag, + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag( + "tags", + "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); + if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); + if (!flags["model"]) throw new TypeError("required option '--model ' not specified"); + + const source = new SourceResolver(io); + const instructions = await source.resolve("instructions", flags["instructions"]); + if (!instructions) { + throw new TypeError("required option '--instructions ' not specified"); + } + const ratingScale = await resolveRatingScale(flags["rating-scale"], source); + if (!ratingScale) { + throw new TypeError("required option '--rating-scale ' not specified"); + } + const tags = parseJsonFlag>( + "tags", + await source.resolve("tags", flags["tags"]), + ); + + const response = await core.eval.createEvaluator( + { + evaluatorName: flags["name"], + level: flags["level"], + evaluatorConfig: { + llmAsAJudge: { + instructions, + ratingScale, + modelConfig: { bedrockEvaluatorModelConfig: { modelId: flags["model"] } }, + }, + }, + kmsKeyArn: flags["kms-key-arn"], + tags, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx index 27dbd0034..2da337170 100644 --- a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx +++ b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx @@ -1,133 +1,12 @@ -import z from "zod"; -import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; +import { Router } from "../../../../router"; import type { AppIO, Core } from "../../../types"; -import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; -import { - RATING_SCALE_PRESET_IDS, - isRatingScalePreset, - ratingScaleFromPreset, -} from "../../ratingScale"; -import { SourceResolver } from "../../source"; - -const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; - -// resolveRatingScale turns the single --rating-scale value into a RatingScale, or -// undefined when the flag is omitted. A value matching a known preset id expands -// to that preset; anything else is a source-aware JSON RatingScale (inline, -// file://, or - for stdin). A file literally named after a preset is still -// reachable via file://. -async function resolveRatingScale( - value: string | undefined, - source: SourceResolver, -): Promise { - if (value === undefined) return undefined; - if (isRatingScalePreset(value)) return ratingScaleFromPreset(value); - const raw = await source.resolve("rating-scale", value); - return parseJsonFlag("rating-scale", raw); +import { createHelpDefault } from "../../../help"; +import { createLlmAsAJudgeCreateHandler } from "./create"; +import { createLlmAsAJudgeUpdateHandler } from "./update"; + +export function createLlmAsAJudgeHandler(core: Core, io: AppIO): Router { + return new Router("llm-as-a-judge", "manage LLM-as-a-Judge evaluators") + .default(createHelpDefault(io)) + .handler(createLlmAsAJudgeCreateHandler(core, io)) + .handler(createLlmAsAJudgeUpdateHandler(core, io)); } - -const ratingScaleFlag = flag( - "rating-scale", - `rating scale: a preset (${RATING_SCALE_PRESET_IDS.join(" | ")}) or a custom RatingScale (JSON inline, file://, or - for stdin)`, - z.string().optional(), -); - -const instructionsFlag = flag( - "instructions", - "evaluation instructions (inline, file://, or - for stdin)", - z.string().optional(), -); - -export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => - createHandler({ - name: "create", - description: "create an LLM-as-a-Judge evaluator", - flags: [ - flag("name", "the name of the evaluator", z.string().optional()), - flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), - flag("model", "the Bedrock model id used to judge", z.string().optional()), - instructionsFlag, - ratingScaleFlag, - flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), - flag( - "tags", - "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", - z.string().optional(), - ), - flag("client-token", "idempotency token", z.string().optional()), - ], - handle: async (ctx, flags) => { - if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); - if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); - if (!flags["model"]) throw new TypeError("required option '--model ' not specified"); - - const source = new SourceResolver(io); - const instructions = await source.resolve("instructions", flags["instructions"]); - if (!instructions) { - throw new TypeError("required option '--instructions ' not specified"); - } - const ratingScale = await resolveRatingScale(flags["rating-scale"], source); - if (!ratingScale) { - throw new TypeError("required option '--rating-scale ' not specified"); - } - const tags = parseJsonFlag>( - "tags", - await source.resolve("tags", flags["tags"]), - ); - - const response = await core.eval.createEvaluator( - { - evaluatorName: flags["name"], - level: flags["level"], - evaluatorConfig: { - llmAsAJudge: { - instructions, - ratingScale, - modelConfig: { bedrockEvaluatorModelConfig: { modelId: flags["model"] } }, - }, - }, - kmsKeyArn: flags["kms-key-arn"], - tags, - clientToken: flags["client-token"], - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); - -export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => - createHandler({ - name: "update", - description: "update an LLM-as-a-Judge evaluator", - flags: [ - flag("id", "the ID of the evaluator to update", z.string().optional()), - instructionsFlag, - flag("model", "the Bedrock model id used to judge", z.string().optional()), - ratingScaleFlag, - flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), - flag("client-token", "idempotency token", z.string().optional()), - ], - handle: async (ctx, flags) => { - if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); - - const source = new SourceResolver(io); - const instructions = await source.resolve("instructions", flags["instructions"]); - const ratingScale = await resolveRatingScale(flags["rating-scale"], source); - - const response = await core.eval.updateLlmAsAJudgeEvaluator( - flags["id"], - { - instructions, - model: flags["model"], - ratingScale, - kmsKeyArn: flags["kms-key-arn"], - clientToken: flags["client-token"], - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx new file mode 100644 index 000000000..300cd53b6 --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx @@ -0,0 +1,41 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { JsonRendererKey } from "../../../../../tui"; +import type { AppIO, Core } from "../../../../types"; +import { coreOptsFromCtx } from "../../../../utils"; +import { SourceResolver } from "../../../source"; +import { instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../utils"; + +export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "update", + description: "update an LLM-as-a-Judge evaluator", + flags: [ + flag("id", "the ID of the evaluator to update", z.string().optional()), + instructionsFlag, + flag("model", "the Bedrock model id used to judge", z.string().optional()), + ratingScaleFlag, + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + + const source = new SourceResolver(io); + const instructions = await source.resolve("instructions", flags["instructions"]); + const ratingScale = await resolveRatingScale(flags["rating-scale"], source); + + const response = await core.eval.updateLlmAsAJudgeEvaluator( + flags["id"], + { + instructions, + model: flags["model"], + ratingScale, + kmsKeyArn: flags["kms-key-arn"], + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx new file mode 100644 index 000000000..65c41b5eb --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx @@ -0,0 +1,39 @@ +import z from "zod"; +import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; +import { flag } from "../../../../router"; +import { parseJsonFlag } from "../../../utils"; +import { + RATING_SCALE_PRESET_IDS, + isRatingScalePreset, + ratingScaleFromPreset, +} from "../../ratingScale"; +import type { SourceResolver } from "../../source"; + +export const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; + +export const instructionsFlag = flag( + "instructions", + "evaluation instructions (inline, file://, or - for stdin)", + z.string().optional(), +); + +export const ratingScaleFlag = flag( + "rating-scale", + `rating scale: a preset (${RATING_SCALE_PRESET_IDS.join(" | ")}) or a custom RatingScale (JSON inline, file://, or - for stdin)`, + z.string().optional(), +); + +// resolveRatingScale turns the single --rating-scale value into a RatingScale, or +// undefined when the flag is omitted. A value matching a known preset id expands +// to that preset; anything else is a source-aware JSON RatingScale (inline, +// file://, or - for stdin). A file literally named after a preset is still +// reachable via file://. +export async function resolveRatingScale( + value: string | undefined, + source: SourceResolver, +): Promise { + if (value === undefined) return undefined; + if (isRatingScalePreset(value)) return ratingScaleFromPreset(value); + const raw = await source.resolve("rating-scale", value); + return parseJsonFlag("rating-scale", raw); +} diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 20308424c..4dc54cce2 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -14,23 +14,23 @@ import type { CoreOptions } from "../../core/types"; // the AgentCore UpdateEvaluator API replaces the whole evaluatorConfig union, and // the llmAsAJudge arm requires instructions + ratingScale + modelConfig together, // so a partial update is only possible by merging over the current definition. -export interface LlmAsAJudgeUpdate { +export type LlmAsAJudgeUpdate = { instructions?: string; model?: string; ratingScale?: RatingScale; kmsKeyArn?: string; clientToken?: string; -} +}; // CodeBasedUpdate carries the fields a caller may change on a code-based // evaluator. Undefined fields are preserved from the existing evaluator, for the // same union-replacement reason described on LlmAsAJudgeUpdate. -export interface CodeBasedUpdate { +export type CodeBasedUpdate = { lambdaArn?: string; timeout?: number; kmsKeyArn?: string; clientToken?: string; -} +}; // CoreEvalClient is the evaluator surface the eval handlers depend on. It is // declared here, next to the handlers that consume it, and implemented by