From 285eab74ff71cac460241470a8177c5993cdda81 Mon Sep 17 00:00:00 2001 From: Raffael Prem Date: Thu, 2 Apr 2026 13:05:35 +0200 Subject: [PATCH] Fixes --- README.md | 38 ++++++++++++++--------------- docs/examples/multi-turn.md | 38 ++++++++++++++--------------- docs/guide/configuration.md | 34 +++++++++++++------------- docs/guide/scenarios.md | 26 ++++++++++---------- docs/guide/trajectory-assertions.md | 2 +- docs/reference/scenario-api.md | 4 +-- 6 files changed, 71 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 994f8da..1e6ac91 100644 --- a/README.md +++ b/README.md @@ -215,31 +215,31 @@ For deterministic tests with predetermined user messages and per-turn assertions // tests/context.sim.ts import { scenario } from '@agentesting/agentest' -scenario('follow-up reuses vehicle context', { +scenario('follow-up reuses order context', { turns: [ { - userMessage: 'How fast was Leo (12345678) last week?', + userMessage: 'What is the status of order ORD-42?', assertions: { toolCalls: { matchMode: 'contains', - expected: [{ name: 'get_speed', args: { id: '12345678' }, argMatchMode: 'partial' }], + expected: [{ name: 'get_order', args: { id: 'ORD-42' }, argMatchMode: 'partial' }], }, }, }, { - userMessage: 'And what about its failure count?', + userMessage: 'And what are the shipping details?', assertions: { toolCalls: { matchMode: 'contains', - expected: [{ name: 'get_failures', args: { id: '12345678' }, argMatchMode: 'partial' }], + expected: [{ name: 'get_shipping', args: { orderId: 'ORD-42' }, argMatchMode: 'partial' }], }, }, }, ], mocks: { tools: { - get_speed: () => ({ speed: 0.8, unit: 'm/s' }), - get_failures: () => ({ count: 5 }), + get_order: () => ({ status: 'shipped', items: 3 }), + get_shipping: () => ({ carrier: 'FedEx', eta: '2024-03-15' }), }, }, }) @@ -1043,8 +1043,8 @@ export default defineConfig({ // Additional named agents agents: { - performance: { type: 'custom', name: 'performance-agent', handler: performanceHandler }, - failure: { type: 'custom', name: 'failure-agent', handler: failureHandler }, + billing: { type: 'custom', name: 'billing-agent', handler: billingHandler }, + support: { type: 'custom', name: 'support-agent', handler: supportHandler }, }, }) ``` @@ -1053,25 +1053,25 @@ Scenarios reference a named agent with the `agent` option: ```ts // Uses the default agent (supervisor) -scenario('routes speed query correctly', { - turns: [{ userMessage: 'How fast was vehicle 12345678?' }], - assertions: { toolCalls: { matchMode: 'contains', expected: [{ name: 'performance_agent' }] } }, +scenario('routes billing query correctly', { + turns: [{ userMessage: 'What is the total for invoice INV-100?' }], + assertions: { toolCalls: { matchMode: 'contains', expected: [{ name: 'get_invoice' }] } }, }) -// Targets the "failure" named agent directly -scenario('failure agent exports to CSV', { - agent: 'failure', - turns: [{ userMessage: 'Export the failure log to CSV' }], +// Targets the "support" named agent directly +scenario('support agent creates a ticket', { + agent: 'support', + turns: [{ userMessage: 'I need help resetting my password' }], assertions: { toolCalls: { matchMode: 'contains', - expected: [{ name: 'get_failure_log' }, { name: 'export_to_csv' }], + expected: [{ name: 'create_ticket' }, { name: 'send_reset_email' }], }, }, mocks: { tools: { - get_failure_log: () => ({ failures: [] }), - export_to_csv: () => ({ fileId: 'abc', url: '/download/abc' }), + create_ticket: () => ({ ticketId: 'TK-001' }), + send_reset_email: () => ({ sent: true }), }, }, }) diff --git a/docs/examples/multi-turn.md b/docs/examples/multi-turn.md index 9a496a6..5de9c78 100644 --- a/docs/examples/multi-turn.md +++ b/docs/examples/multi-turn.md @@ -128,26 +128,26 @@ This is ideal for testing **context carry-forward**, **conversation continuity** ```ts import { scenario } from '@agentesting/agentest' -scenario('follow-up reuses vehicle context', { +scenario('follow-up reuses order context', { turns: [ { - userMessage: 'How fast was Leo (12345678) last week?', + userMessage: 'What is the status of order ORD-42?', assertions: { toolCalls: { matchMode: 'contains', expected: [ - { name: 'performance_agent', args: { serials: '12345678' }, argMatchMode: 'partial' }, + { name: 'get_order', args: { id: 'ORD-42' }, argMatchMode: 'partial' }, ], }, }, }, { - userMessage: 'And what about its failure count in the same period?', + userMessage: 'And what are the shipping details for that order?', assertions: { toolCalls: { matchMode: 'contains', expected: [ - { name: 'failure_agent', args: { serials: '12345678' }, argMatchMode: 'partial' }, + { name: 'get_shipping', args: { orderId: 'ORD-42' }, argMatchMode: 'partial' }, ], }, }, @@ -156,14 +156,14 @@ scenario('follow-up reuses vehicle context', { mocks: { tools: { - performance_agent: () => ({ speed: 0.8, unit: 'm/s' }), - failure_agent: () => ({ count: 5, severity_breakdown: { warning: 3, error: 2 } }), + get_order: () => ({ status: 'shipped', items: 3, total: 89.99 }), + get_shipping: () => ({ carrier: 'FedEx', eta: '2024-03-15', trackingId: 'FX-99887' }), }, }, }) ``` -The second turn says "its failure count" — the agent must carry forward that "it" refers to vehicle `12345678` from the previous turn. The per-turn assertion verifies this. +The second turn says "that order" — the agent must carry forward that it refers to order `ORD-42` from the previous turn. The per-turn assertion verifies this. ### Example: Domain Switch with Follow-up Export @@ -171,21 +171,21 @@ The second turn says "its failure count" — the agent must carry forward that " scenario('cross-domain pivot then export', { turns: [ { - userMessage: 'What was the energy consumption of Leo (12345678) from Jan 1 to Jan 7?', + userMessage: 'Show me the sales for product SKU-200 from Jan 1 to Jan 7.', assertions: { toolCalls: { matchMode: 'contains', - expected: [{ name: 'performance_agent', argMatchMode: 'ignore' }], + expected: [{ name: 'get_sales', argMatchMode: 'ignore' }], }, }, }, { - userMessage: 'Were there any errors during that period?', + userMessage: 'Were there any returns during that period?', assertions: { toolCalls: { matchMode: 'contains', expected: [ - { name: 'failure_agent', args: { serials: '12345678' }, argMatchMode: 'partial' }, + { name: 'get_returns', args: { productId: 'SKU-200' }, argMatchMode: 'partial' }, ], }, }, @@ -203,8 +203,8 @@ scenario('cross-domain pivot then export', { mocks: { tools: { - performance_agent: () => ({ energy: 12.4, unit: 'kWh' }), - failure_agent: () => ({ count: 3 }), + get_sales: () => ({ units: 124, revenue: 4960.00 }), + get_returns: () => ({ count: 3, refunded: 120.00 }), export_to_csv: () => ({ fileId: 'export-001', url: '/download/export-001' }), }, }, @@ -228,17 +228,17 @@ To enable LLM-as-judge evaluation on scripted scenarios, provide a `goal`: ```ts scenario('follow-up with quality evaluation', { - goal: 'Get speed and failure data for vehicle Leo.', + goal: 'Get order status and shipping details for order ORD-42.', turns: [ - { userMessage: 'How fast was Leo (12345678) last week?' }, - { userMessage: 'And what about its failure count?' }, + { userMessage: 'What is the status of order ORD-42?' }, + { userMessage: 'And what are the shipping details?' }, ], mocks: { tools: { - performance_agent: () => ({ speed: 0.8 }), - failure_agent: () => ({ count: 5 }), + get_order: () => ({ status: 'shipped', items: 3 }), + get_shipping: () => ({ carrier: 'FedEx', eta: '2024-03-15' }), }, }, }) diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index c1cf43f..cf8ea24 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -422,15 +422,15 @@ export default defineConfig({ // Additional named agents agents: { - performance: { + billing: { type: 'custom', - name: 'performance-agent', - handler: performanceHandler, + name: 'billing-agent', + handler: billingHandler, }, - failure: { + support: { type: 'custom', - name: 'failure-agent', - handler: failureHandler, + name: 'support-agent', + handler: supportHandler, }, }, }) @@ -440,33 +440,33 @@ Scenarios reference a named agent with the `agent` option: ```ts // Uses the default agent (supervisor) -scenario('routes speed queries to performance agent', { - turns: [{ userMessage: 'How fast was vehicle 12345678?' }], +scenario('routes billing query correctly', { + turns: [{ userMessage: 'What is the total for invoice INV-100?' }], assertions: { toolCalls: { matchMode: 'contains', - expected: [{ name: 'performance_agent' }], + expected: [{ name: 'get_invoice' }], }, }, }) -// Uses the named "failure" agent directly -scenario('failure agent exports to CSV', { - agent: 'failure', - turns: [{ userMessage: 'Export the failure log to CSV' }], +// Uses the named "support" agent directly +scenario('support agent creates a ticket', { + agent: 'support', + turns: [{ userMessage: 'I need help resetting my password' }], assertions: { toolCalls: { matchMode: 'contains', expected: [ - { name: 'get_failure_log', argMatchMode: 'ignore' }, - { name: 'export_to_csv', argMatchMode: 'ignore' }, + { name: 'create_ticket', argMatchMode: 'ignore' }, + { name: 'send_reset_email', argMatchMode: 'ignore' }, ], }, }, mocks: { tools: { - get_failure_log: () => ({ failures: [...] }), - export_to_csv: () => ({ fileId: 'abc', url: '...' }), + create_ticket: () => ({ ticketId: 'TK-001' }), + send_reset_email: () => ({ sent: true }), }, }, }) diff --git a/docs/guide/scenarios.md b/docs/guide/scenarios.md index 8f62ea1..ec27518 100644 --- a/docs/guide/scenarios.md +++ b/docs/guide/scenarios.md @@ -20,8 +20,8 @@ scenario('simulated booking', { ```ts scenario('scripted follow-up', { turns: [ - { userMessage: 'How fast was Leo (12345678) last week?' }, - { userMessage: 'And what about its failure count?' }, + { userMessage: 'What is the status of order ORD-42?' }, + { userMessage: 'And what are the shipping details?' }, ], }) ``` @@ -389,15 +389,15 @@ export default defineConfig({ // scenarios/routing.sim.ts import { scenario } from '@agentesting/agentest' -scenario('supervisor routes to performance agent', { +scenario('supervisor routes to billing agent', { turns: [ { - userMessage: 'How fast was vehicle 12345678 last week?', + userMessage: 'What is the total for invoice INV-100?', assertions: { toolCalls: { matchMode: 'contains', expected: [ - { name: 'get_report_data', args: { serials: '12345678' }, argMatchMode: 'partial' }, + { name: 'get_invoice', args: { id: 'INV-100' }, argMatchMode: 'partial' }, ], }, }, @@ -406,13 +406,13 @@ scenario('supervisor routes to performance agent', { mocks: { tools: { - get_report_data: (args) => ({ speed_avg: 0.8, unit: 'm/s' }), + get_invoice: (args) => ({ total: 249.99, currency: 'USD', status: 'paid' }), }, }, }) ``` -When the agent internally calls `get_report_data`, the mock client calls `ctx.resolveTool('get_report_data', args)`, which: +When the agent internally calls `get_invoice`, the mock client calls `ctx.resolveTool('get_invoice', args)`, which: 1. Resolves through agentest's per-scenario mock definitions 2. Records the tool call for trajectory assertions 3. Returns the mock result to the agent @@ -425,17 +425,17 @@ When your config defines [named agents](/guide/configuration#named-agents), scen ```ts // Uses the default agent -scenario('supervisor routes to performance', { +scenario('supervisor routes billing query', { turns: [ - { userMessage: 'How fast was vehicle 12345678 last week?' }, + { userMessage: 'What is the total for invoice INV-100?' }, ], }) -// Targets the "failure" named agent -scenario('failure agent calls export_to_csv', { - agent: 'failure', +// Targets the "support" named agent +scenario('support agent creates a ticket', { + agent: 'support', turns: [ - { userMessage: 'Export the failure log to CSV' }, + { userMessage: 'I need help resetting my password' }, ], }) ``` diff --git a/docs/guide/trajectory-assertions.md b/docs/guide/trajectory-assertions.md index 1f902a3..faee3c7 100644 --- a/docs/guide/trajectory-assertions.md +++ b/docs/guide/trajectory-assertions.md @@ -40,7 +40,7 @@ scenario('user books a morning slot', { Scenario-level assertions are checked **per-conversation** across all turns combined. If any conversation fails the assertion, the entire scenario fails. ::: tip Per-turn Assertions -When using [scripted multi-turn scenarios](/examples/multi-turn#scripted-multi-turn-deterministic), each turn can define its own trajectory assertions that are checked against only that turn's tool calls. This is useful for verifying context carry-forward — e.g., that a follow-up question routes to the right domain agent with the correct vehicle serial from the previous turn. +When using [scripted multi-turn scenarios](/examples/multi-turn#scripted-multi-turn-deterministic), each turn can define its own trajectory assertions that are checked against only that turn's tool calls. This is useful for verifying context carry-forward — e.g., that a follow-up question routes to the right domain agent with the correct order ID from the previous turn. ::: ## Match Modes diff --git a/docs/reference/scenario-api.md b/docs/reference/scenario-api.md index 3488e7f..1996414 100644 --- a/docs/reference/scenario-api.md +++ b/docs/reference/scenario-api.md @@ -45,11 +45,11 @@ Each turn can include per-turn trajectory assertions. ```ts turns: [ { - userMessage: 'How fast was vehicle 12345678?', + userMessage: 'What is the status of order ORD-42?', assertions: { toolCalls: { matchMode: 'contains', - expected: [{ name: 'performance_agent', argMatchMode: 'ignore' }], + expected: [{ name: 'get_order', argMatchMode: 'ignore' }], }, }, },