feat(eval): add eval evaluator CLI commands (CRUD + LLaJ/code-based)

README.md

@@ -64,6 +64,17 @@ agentcore                          # interactive TUI
 │   └── endpoint
 │       ├── get                    # get a Runtime endpoint by qualifier
 │       └── list                   # list a Runtime's endpoints
+├── eval                           # evaluate and optimize AgentCore agents
+│   └── evaluator                  # manage AgentCore evaluators
+│       ├── llm-as-a-judge         # LLM-as-a-Judge evaluators
+│       │   ├── create             # create (instructions + rating scale + model)
+│       │   └── update             # update (merged over the existing config)
+│       ├── code-based             # code-based (Lambda-backed) evaluators
+│       │   ├── create             # create (Lambda ARN + optional timeout)
+│       │   └── update             # update (merged over the existing config)
+│       ├── get                    # get an evaluator by id (type-agnostic)
+│       ├── list                   # list evaluators (client-side --type filter)
+│       └── delete                 # delete an evaluator by id (requires --yes)
 └── config                         # read/write global config values
 ```
 
@@ -114,8 +125,35 @@ agentcore identity api-key-credential-provider get --name my-provider
 agentcore identity api-key-credential-provider list --max-results 10
 agentcore identity api-key-credential-provider update --name my-provider --api-key <new-key>
 agentcore identity api-key-credential-provider delete --name my-provider
+
+# Manage evaluators
+# Create an LLM-as-a-Judge evaluator with a rating-scale preset.
+agentcore eval evaluator llm-as-a-judge create \
+  --name order-support-quality \
+  --level SESSION \
+  --model us.anthropic.claude-sonnet-4-5 \
+  --instructions "Judge whether the order-support agent answered correctly." \
+  --rating-scale 1-5-quality \
+  --json
+
+# Create a code-based (Lambda-backed) evaluator; timeout defaults to the service value.
+agentcore eval evaluator code-based create \
+  --name refund-policy-compliance \
+  --level SESSION \
+  --lambda-arn arn:aws:lambda:us-west-2:123456789012:function:refund-policy \
+  --json
+
+# Get, list, delete. --type filters the returned page (builtin | code-based | llm-as-a-judge).
+agentcore eval evaluator get --id <evaluatorId> --json
+agentcore eval evaluator list --type llm-as-a-judge --max-results 20 --json
+agentcore eval evaluator delete --id <evaluatorId> --yes --json
 ```
 
+Source-aware values: any field flag documented as such accepts the value inline,
+`file://<path>` to read it from a file, or `-` to read it from stdin (the AWS CLI
+`file://` convention). A command reads stdin from at most one flag. For example,
+`--instructions file://order-quality.txt` or `--instructions -`.
+
 Bare Runtime branches and leaves require a TTY on stdin and stdout. Supplying
 operation flags runs the command headlessly, and `--json` always suppresses TUI
 rendering.

src/core/eval.test.ts

@@ -0,0 +1,102 @@
+import { test, expect } from "bun:test";
+import {
+  GetEvaluatorCommand,
+  UpdateEvaluatorCommand,
+  type GetEvaluatorResponse,
+} from "@aws-sdk/client-bedrock-agentcore-control";
+import { EvalClient } from "./eval";
+import type { AwsClients, CoreOptions } from "./types";
+
+const OPTIONS: CoreOptions = { region: "us-west-2", endpointUrl: undefined };
+
+// fakeClients returns an AwsClients whose control().send dispatches GetEvaluator
+// to `getResponse` and records UpdateEvaluator inputs into `updates`. Only
+// control() is used by EvalClient; data()/iam() throw if ever reached.
+function fakeClients(getResponse: GetEvaluatorResponse, updates: unknown[]): AwsClients {
+  const control = {
+    send: async (command: { input: unknown }) => {
+      if (command instanceof GetEvaluatorCommand) return getResponse;
+      if (command instanceof UpdateEvaluatorCommand) {
+        updates.push(command.input);
+        return { evaluatorId: "e-1" };
+      }
+      throw new Error(`unexpected command ${command.constructor.name}`);
+    },
+  };
+  return {
+    control: () => control as never,
+    data: () => {
+      throw new Error("data client not expected");
+    },
+    iam: () => {
+      throw new Error("iam client not expected");
+    },
+  };
+}
+
+test("updateLlmAsAJudgeEvaluator preserves existing fields not passed", async () => {
+  const updates: unknown[] = [];
+  const current: GetEvaluatorResponse = {
+    evaluatorConfig: {
+      llmAsAJudge: {
+        instructions: "old instructions",
+        ratingScale: { numerical: [{ value: 1, label: "L", definition: "d" }] },
+        modelConfig: { bedrockEvaluatorModelConfig: { modelId: "old-model" } },
+      },
+    },
+  } as GetEvaluatorResponse;
+
+  const client = new EvalClient(fakeClients(current, updates));
+  // Only the model changes; instructions + ratingScale should be carried over.
+  await client.updateLlmAsAJudgeEvaluator("e-1", { model: "new-model" }, OPTIONS);
+
+  expect(updates).toHaveLength(1);
+  const sent = updates[0] as any;
+  expect(sent.evaluatorConfig.llmAsAJudge.modelConfig.bedrockEvaluatorModelConfig.modelId).toBe(
+    "new-model",
+  );
+  expect(sent.evaluatorConfig.llmAsAJudge.instructions).toBe("old instructions");
+  expect(sent.evaluatorConfig.llmAsAJudge.ratingScale.numerical[0].label).toBe("L");
+});
+
+test("updateLlmAsAJudgeEvaluator rejects a non-LLM evaluator", async () => {
+  const current: GetEvaluatorResponse = {
+    evaluatorConfig: { codeBased: { lambdaConfig: { lambdaArn: "arn:x" } } },
+  } as GetEvaluatorResponse;
+  const client = new EvalClient(fakeClients(current, []));
+  await expect(
+    client.updateLlmAsAJudgeEvaluator("e-1", { instructions: "x" }, OPTIONS),
+  ).rejects.toThrow(/not an LLM-as-a-Judge evaluator/);
+});
+
+test("updateCodeBasedEvaluator preserves the lambda ARN when only timeout changes", async () => {
+  const updates: unknown[] = [];
+  const current: GetEvaluatorResponse = {
+    evaluatorConfig: {
+      codeBased: { lambdaConfig: { lambdaArn: "arn:keep", lambdaTimeoutInSeconds: 10 } },
+    },
+  } as GetEvaluatorResponse;
+
+  const client = new EvalClient(fakeClients(current, updates));
+  await client.updateCodeBasedEvaluator("e-1", { timeout: 45 }, OPTIONS);
+
+  const sent = updates[0] as any;
+  expect(sent.evaluatorConfig.codeBased.lambdaConfig.lambdaArn).toBe("arn:keep");
+  expect(sent.evaluatorConfig.codeBased.lambdaConfig.lambdaTimeoutInSeconds).toBe(45);
+});
+
+test("updateCodeBasedEvaluator rejects a non-code-based evaluator", async () => {
+  const current = {
+    evaluatorConfig: {
+      llmAsAJudge: {
+        instructions: "i",
+        ratingScale: { numerical: [] },
+        modelConfig: { bedrockEvaluatorModelConfig: { modelId: "m" } },
+      },
+    },
+  } as unknown as GetEvaluatorResponse;
+  const client = new EvalClient(fakeClients(current, []));
+  await expect(client.updateCodeBasedEvaluator("e-1", { timeout: 30 }, OPTIONS)).rejects.toThrow(
+    /not a code-based evaluator/,
+  );
+});

src/core/eval.tsx

@@ -0,0 +1,138 @@
+import {
+  CreateEvaluatorCommand,
+  DeleteEvaluatorCommand,
+  GetEvaluatorCommand,
+  ListEvaluatorsCommand,
+  UpdateEvaluatorCommand,
+  type CreateEvaluatorRequest,
+  type CreateEvaluatorResponse,
+  type DeleteEvaluatorResponse,
+  type EvaluatorConfig,
+  type GetEvaluatorResponse,
+  type ListEvaluatorsResponse,
+  type UpdateEvaluatorResponse,
+} from "@aws-sdk/client-bedrock-agentcore-control";
+import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types";
+import type { AwsClients, CoreOptions } from "./types";
+import { toClientConfig } from "./utils";
+
+export class EvalClient implements CoreEvalClient {
+  constructor(private readonly clients: AwsClients) {}
+
+  async createEvaluator(
+    request: CreateEvaluatorRequest,
+    options: CoreOptions,
+  ): Promise<CreateEvaluatorResponse> {
+    return this.clients.control(toClientConfig(options)).send(new CreateEvaluatorCommand(request));
+  }
+
+  // updateLlmAsAJudgeEvaluator rebuilds the full llmAsAJudge config from the
+  // current evaluator, overlays the provided fields, and sends it. UpdateEvaluator
+  // replaces the entire evaluatorConfig union, and the llmAsAJudge arm requires
+  // instructions + ratingScale + modelConfig together, so a partial update would
+  // otherwise drop the fields the caller didn't pass.
+  async updateLlmAsAJudgeEvaluator(
+    id: string,
+    update: LlmAsAJudgeUpdate,
+    options: CoreOptions,
+  ): Promise<UpdateEvaluatorResponse> {
+    const control = this.clients.control(toClientConfig(options));
+    const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id }));
+    const existing =
+      current.evaluatorConfig && "llmAsAJudge" in current.evaluatorConfig
+        ? current.evaluatorConfig.llmAsAJudge
+        : undefined;
+
+    const instructions = update.instructions ?? existing?.instructions;
+    const ratingScale = update.ratingScale ?? existing?.ratingScale;
+    const modelId =
+      update.model ??
+      (existing?.modelConfig && "bedrockEvaluatorModelConfig" in existing.modelConfig
+        ? existing.modelConfig.bedrockEvaluatorModelConfig?.modelId
+        : undefined);
+
+    if (!instructions || !ratingScale || !modelId) {
+      throw new TypeError(
+        `Evaluator "${id}" is not an LLM-as-a-Judge evaluator or is missing configuration required to update it`,
+      );
+    }
+
+    const evaluatorConfig: EvaluatorConfig = {
+      llmAsAJudge: {
+        instructions,
+        ratingScale,
+        modelConfig: { bedrockEvaluatorModelConfig: { modelId } },
+      },
+    };
+
+    return control.send(
+      new UpdateEvaluatorCommand({
+        evaluatorId: id,
+        evaluatorConfig,
+        kmsKeyArn: update.kmsKeyArn,
+        clientToken: update.clientToken,
+      }),
+    );
+  }
+
+  // updateCodeBasedEvaluator mirrors updateLlmAsAJudgeEvaluator: it merges the
+  // provided lambda ARN / timeout over the current codeBased config so unset
+  // fields are preserved across the union-replacing UpdateEvaluator call.
+  async updateCodeBasedEvaluator(
+    id: string,
+    update: CodeBasedUpdate,
+    options: CoreOptions,
+  ): Promise<UpdateEvaluatorResponse> {
+    const control = this.clients.control(toClientConfig(options));
+    const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id }));
+    const existing =
+      current.evaluatorConfig && "codeBased" in current.evaluatorConfig
+        ? current.evaluatorConfig.codeBased
+        : undefined;
+    const existingLambda =
+      existing && "lambdaConfig" in existing ? existing.lambdaConfig : undefined;
+
+    const lambdaArn = update.lambdaArn ?? existingLambda?.lambdaArn;
+    if (!lambdaArn) {
+      throw new TypeError(
+        `Evaluator "${id}" is not a code-based evaluator or is missing configuration required to update it`,
+      );
+    }
+    const lambdaTimeoutInSeconds = update.timeout ?? existingLambda?.lambdaTimeoutInSeconds;
+
+    const evaluatorConfig: EvaluatorConfig = {
+      codeBased: { lambdaConfig: { lambdaArn, lambdaTimeoutInSeconds } },
+    };
+
+    return control.send(
+      new UpdateEvaluatorCommand({
+        evaluatorId: id,
+        evaluatorConfig,
+        kmsKeyArn: update.kmsKeyArn,
+        clientToken: update.clientToken,
+      }),
+    );
+  }
+
+  async getEvaluator(id: string, options: CoreOptions): Promise<GetEvaluatorResponse> {
+    return this.clients
+      .control(toClientConfig(options))
+      .send(new GetEvaluatorCommand({ evaluatorId: id }));
+  }
+
+  async listEvaluators(
+    nextToken: string | undefined,
+    maxResults: number | undefined,
+    options: CoreOptions,
+  ): Promise<ListEvaluatorsResponse> {
+    return this.clients
+      .control(toClientConfig(options))
+      .send(new ListEvaluatorsCommand({ nextToken, maxResults }));
+  }
+
+  async deleteEvaluator(id: string, options: CoreOptions): Promise<DeleteEvaluatorResponse> {
+    return this.clients
+      .control(toClientConfig(options))
+      .send(new DeleteEvaluatorCommand({ evaluatorId: id }));
+  }
+}

src/core/index.tsx

@@ -1,6 +1,7 @@
 import { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control";
 import { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore";
 import { IAMClient } from "@aws-sdk/client-iam";
+import { EvalClient } from "./eval";
 import { HarnessClient } from "./harness";
 import { IdentityClient } from "./identity";
 import { RuntimeClient } from "./runtime";
@@ -48,6 +49,7 @@ export class CoreClient implements AwsClients {
   readonly harness: HarnessClient = new HarnessClient(this);
   readonly identity: IdentityClient = new IdentityClient(this);
   readonly runtime: RuntimeClient = new RuntimeClient(this);
+  readonly eval: EvalClient = new EvalClient(this);
 
   readonly projectManager: ProjectManager;