diff --git a/.gitignore b/.gitignore index 86e12d3..7306fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ demos/**/.env demos/**/__pycache__/ demos/**/*.pyc demos/**/.venv/ + +agentesting-agentest-* diff --git a/README.md b/README.md index aa74408..51785e5 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ npm install @agentesting/agentest --save-dev | LLM-as-judge metrics | Helpfulness, coherence, relevance, faithfulness, goal completion, behavior failure detection | | Comparison mode | Run the same scenarios against multiple models/configs side-by-side | | CI-ready CLI | Exit codes, JSON reporter, GitHub Actions annotations, watch mode | +| Named agents | Define multiple agents (supervisor + domain) and target them per-scenario | | Custom handler | Bring any agent — HTTP endpoint or in-process function. Works with any framework | --- @@ -1022,6 +1023,57 @@ YAML configs go through the same validation and environment variable interpolati --- +## Named Agents + +For multi-agent architectures, define multiple named agents and target them per-scenario. This lets you test both high-level routing (supervisor) and low-level tool usage (domain agents) in one test suite. + +```ts +export default defineConfig({ + // Default agent — used when scenario doesn't specify one + agent: { + type: 'custom', + name: 'supervisor', + handler: supervisorHandler, + }, + + // Additional named agents + agents: { + performance: { type: 'custom', name: 'performance-agent', handler: performanceHandler }, + failure: { type: 'custom', name: 'failure-agent', handler: failureHandler }, + }, +}) +``` + +Scenarios reference a named agent with the `agent` option: + +```ts +// Uses the default agent (supervisor) +scenario('routes speed query correctly', { + turns: [{ userMessage: 'How fast was vehicle 12345678?' }], + assertions: { toolCalls: { matchMode: 'contains', expected: [{ name: 'performance_agent' }] } }, +}) + +// Targets the "failure" named agent directly +scenario('failure agent exports to CSV', { + agent: 'failure', + turns: [{ userMessage: 'Export the failure log to CSV' }], + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [{ name: 'get_failure_log' }, { name: 'export_to_csv' }], + }, + }, + mocks: { + tools: { + get_failure_log: () => ({ failures: [] }), + export_to_csv: () => ({ fileId: 'abc', url: '/download/abc' }), + }, + }, +}) +``` + +--- + ## Comparison Mode Run the same scenarios against multiple models or agent configurations side-by-side. Add `compare` to your config — each entry overrides specific fields from the primary `agent`: diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 8dbcdd7..e03a11f 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -405,6 +405,92 @@ unmockedTools: 'passthrough', // return undefined (no-op) See [Mocks Guide](/guide/mocks) for more details. +## Named Agents + +For multi-agent architectures, you can define multiple named agents and target them from individual scenarios. This lets you test both high-level routing (supervisor) and low-level tool usage (domain agents) in the same test suite. + +```ts +export default defineConfig({ + // Default agent — used when scenario doesn't specify one + agent: { + type: 'custom', + name: 'supervisor', + handler: supervisorHandler, + }, + + // Additional named agents + agents: { + performance: { + type: 'custom', + name: 'performance-agent', + handler: performanceHandler, + }, + failure: { + type: 'custom', + name: 'failure-agent', + handler: failureHandler, + }, + }, +}) +``` + +Scenarios reference a named agent with the `agent` option: + +```ts +// Uses the default agent (supervisor) +scenario('routes speed queries to performance agent', { + turns: [{ userMessage: 'How fast was vehicle 12345678?' }], + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [{ name: 'performance_agent' }], + }, + }, +}) + +// Uses the named "failure" agent directly +scenario('failure agent exports to CSV', { + agent: 'failure', + turns: [{ userMessage: 'Export the failure log to CSV' }], + assertions: { + toolCalls: { + matchMode: 'contains', + expected: [ + { name: 'get_failure_log', argMatchMode: 'ignore' }, + { name: 'export_to_csv', argMatchMode: 'ignore' }, + ], + }, + }, + mocks: { + tools: { + get_failure_log: () => ({ failures: [...] }), + export_to_csv: () => ({ fileId: 'abc', url: '...' }), + }, + }, +}) +``` + +### When to Use Named Agents + +Named agents are ideal for **multi-agent architectures** where a supervisor routes to specialized sub-agents: + +- **Supervisor scenarios** (default agent): Test routing — does the right domain agent get called with the right parameters? +- **Domain agent scenarios** (named agents): Test reasoning within a domain — does the agent call the right inner tools? Does it handle edge cases? + +This separation keeps tests focused and fast. Supervisor tests mock domain agents as black boxes. Domain tests mock only the data-fetching tools underneath. + +Named agents also work with HTTP endpoints — you can define different endpoints or headers per agent: + +```ts +agents: { + staging: { + name: 'staging-api', + endpoint: 'https://staging.example.com/api/chat', + headers: { Authorization: 'Bearer ${STAGING_KEY}' }, + }, +} +``` + ## Comparison Mode Run the same scenarios against multiple models or agent configurations side-by-side: diff --git a/docs/guide/scenarios.md b/docs/guide/scenarios.md index c786b30..8f62ea1 100644 --- a/docs/guide/scenarios.md +++ b/docs/guide/scenarios.md @@ -419,6 +419,29 @@ When the agent internally calls `get_report_data`, the mock client calls `ctx.re This gives you full control over tool responses while testing the actual routing logic of your supervisor. +## Targeting Named Agents + +When your config defines [named agents](/guide/configuration#named-agents), scenarios can target a specific agent with the `agent` option: + +```ts +// Uses the default agent +scenario('supervisor routes to performance', { + turns: [ + { userMessage: 'How fast was vehicle 12345678 last week?' }, + ], +}) + +// Targets the "failure" named agent +scenario('failure agent calls export_to_csv', { + agent: 'failure', + turns: [ + { userMessage: 'Export the failure log to CSV' }, + ], +}) +``` + +This is useful for multi-agent architectures where you want to test both routing (at the supervisor level) and inner tool usage (at the domain agent level) in the same test suite. + ## Complete Example See [Basic Scenario Example](/examples/basic-scenario) for a full walkthrough. diff --git a/docs/public/llms.txt b/docs/public/llms.txt index e46dbb5..9f54f19 100644 --- a/docs/public/llms.txt +++ b/docs/public/llms.txt @@ -10,6 +10,7 @@ Agentest is an embedded agent simulation and evaluation framework for Node.js/Ty - **Tool-call mocks**: Intercept and control tool calls with functions, sequences, and error simulation - **Trajectory assertions**: Verify tool call order and arguments with strict, contains, unordered, and within match modes - **LLM-as-judge metrics**: Helpfulness, coherence, relevance, faithfulness, goal completion, behavior failure detection (8 built-in metrics + custom) +- **Named agents**: Define multiple agents (supervisor + domain) and target them per-scenario for multi-agent architectures - **Comparison mode**: Run the same scenarios against multiple models/configs side-by-side - **CI-ready CLI**: Exit codes, JSON reporter, GitHub Actions annotations, watch mode - **Custom handler**: Bring any agent — HTTP endpoint or in-process function. Works with any framework diff --git a/package.json b/package.json index 00ef567..f235cd4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agentesting/agentest", - "version": "0.0.14", + "version": "0.0.15", "description": "Embedded agent simulation & evaluation framework for Node.js/TypeScript", "type": "module", "main": "./dist/index.cjs", @@ -10,7 +10,7 @@ ], "types": "./dist/index.d.cts", "bin": { - "agentest": "./dist/cli.js" + "agentest": "dist/cli.js" }, "exports": { ".": { diff --git a/src/config/defineConfig.ts b/src/config/defineConfig.ts index bb54e49..5281f92 100644 --- a/src/config/defineConfig.ts +++ b/src/config/defineConfig.ts @@ -29,6 +29,19 @@ export function defineConfig(input: AgentestConfigInput): AgentestConfig { result.agent = { ...result.agent, headers: interpolateEnvVars(result.agent.headers) } } + // Interpolate env vars in named agent headers + if (result.agents) { + const interpolated: typeof result.agents = {} + for (const [key, agent] of Object.entries(result.agents)) { + if (agent.type !== 'custom' && agent.headers) { + interpolated[key] = { ...agent, headers: interpolateEnvVars(agent.headers) } + } else { + interpolated[key] = agent + } + } + result.agents = interpolated + } + // Interpolate env vars in compare entry headers if (result.compare) { result.compare = result.compare.map((entry) => diff --git a/src/config/schema.ts b/src/config/schema.ts index f0f4cca..49cd169 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -109,6 +109,7 @@ export const thresholdsSchema = z export const configSchema = z.object({ agent: agentConfigSchema, + agents: z.record(agentConfigSchema).optional(), model: z.string().default(DEFAULT_MODEL), provider: z diff --git a/src/scenario/types.ts b/src/scenario/types.ts index c16f7a5..1d66f04 100644 --- a/src/scenario/types.ts +++ b/src/scenario/types.ts @@ -37,6 +37,9 @@ export interface ScriptedTurn { } export interface ScenarioOptions { + /** Named agent key from config.agents to use instead of the default agent. */ + agent?: string + profile?: string goal?: string knowledge?: KnowledgeItem[] @@ -58,6 +61,12 @@ export interface ScenarioOptions { } export function validateScenarioOptions(options: ScenarioOptions): void { + if (options.agent !== undefined) { + if (typeof options.agent !== 'string' || !options.agent.trim()) { + throw new Error('Scenario "agent" must be a non-empty string referencing a named agent') + } + } + const isScripted = Array.isArray(options.turns) if (isScripted) { diff --git a/src/simulator/simulator.ts b/src/simulator/simulator.ts index e359868..9734978 100644 --- a/src/simulator/simulator.ts +++ b/src/simulator/simulator.ts @@ -74,9 +74,26 @@ export class Simulator { (isScripted ? 1 : this.config.conversationsPerScenario) const conversations: ConversationRecord[] = [] + // Resolve per-scenario agent override + let agentClient = this.agentClient + if (scenario.options.agent) { + const namedAgent = this.config.agents?.[scenario.options.agent] + if (!namedAgent) { + throw new Error( + `Scenario "${scenario.name}" references agent "${scenario.options.agent}" ` + + `but no such agent was found in config.agents. ` + + `Available agents: ${this.config.agents ? Object.keys(this.config.agents).join(', ') : '(none)'}`, + ) + } + agentClient = new AgentClient({ + ...this.config, + agent: namedAgent, + }) + } + for (let i = 0; i < conversationCount; i++) { const conversationId = `conv-${i + 1}-${randomUUID().slice(0, 8)}` - const record = await this.runConversation(scenario, conversationId) + const record = await this.runConversation(scenario, conversationId, agentClient) conversations.push(record) } @@ -86,6 +103,7 @@ export class Simulator { private async runConversation( scenario: Scenario, conversationId: string, + agentClient?: AgentClient, ): Promise { const isScripted = Array.isArray(scenario.options.turns) && scenario.options.turns.length > 0 const scriptedTurns = scenario.options.turns @@ -101,6 +119,7 @@ export class Simulator { ) mockResolver.reset() + const client = agentClient ?? this.agentClient const conversationHistory: Array<{ role: 'user' | 'assistant'; content: string }> = [] const agentMessages: ChatMessage[] = [] const turns: TurnRecord[] = [] @@ -152,7 +171,7 @@ export class Simulator { conversationId, scenarioName: scenario.name, } - this.agentClient.setHandlerContext(handlerCtx) + client.setHandlerContext(handlerCtx) this.onProgress?.({ scenario: scenario.name, @@ -163,7 +182,7 @@ export class Simulator { }) const MAX_TOOL_CALL_ROUNDS = 50 let toolCallRound = 0 - let response = await this.agentClient.send(agentMessages) + let response = await client.send(agentMessages) // Tool call loop — keep going until agent returns text without tool calls while (response.hasToolCalls) { @@ -194,7 +213,7 @@ export class Simulator { } // Send back to agent with tool results - response = await this.agentClient.send(agentMessages) + response = await client.send(agentMessages) } // Agent returned a text response (no tool calls)