From c3cdd28b66122ab1626db2c3e754c0ed4e938fae Mon Sep 17 00:00:00 2001 From: Raffael Prem Date: Tue, 31 Mar 2026 11:19:37 +0200 Subject: [PATCH] Add file level filtering --- README.md | 5 +++++ docs/guide/configuration.md | 2 ++ docs/reference/cli.md | 8 ++++++++ docs/reference/config-api.md | 8 ++++++++ package.json | 2 +- src/cli.ts | 10 +++++++++- 6 files changed, 33 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 51785e5..994f8da 100644 --- a/README.md +++ b/README.md @@ -444,6 +444,7 @@ providerOptions: { conversationsPerScenario: 3, // run 3 independent conversations per scenario maxTurns: 8, // max user↔agent exchanges per conversation concurrency: 20, // max parallel LLM calls across all scenarios +debounceMs: 0, // optional delay (ms) between scenario starts ``` | Option | Type | Default | Description | @@ -451,6 +452,7 @@ concurrency: 20, // max parallel LLM calls across all scenarios | `conversationsPerScenario` | `number` | `3` | How many independent conversations to run per scenario. More conversations = more statistical confidence, but more LLM cost | | `maxTurns` | `number` | `8` | Upper bound on user↔agent exchanges. The simulated user can stop earlier if the goal is met | | `concurrency` | `number` | `20` | Max parallel LLM calls. Controls both simulation and evaluation parallelism. Lower this if you're hitting rate limits | +| `debounceMs` | `number` | `0` | Minimum delay in milliseconds between scenario starts. Useful for avoiding rate limits when running many scenarios | Both `conversationsPerScenario` and `maxTurns` can be overridden per-scenario. @@ -960,6 +962,9 @@ npx agentest run --cwd ./packages/my-agent npx agentest run --scenario "booking" npx agentest run --scenario "cancel" +# Filter by file path (case-insensitive substring match) +npx agentest run --file "auth.sim.ts" + # Print full conversation transcripts npx agentest run --verbose diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index e03a11f..c1cf43f 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -281,6 +281,7 @@ Control how many conversations to run and how they're executed: conversationsPerScenario: 3, // run 3 independent conversations per scenario maxTurns: 8, // max user↔agent exchanges per conversation concurrency: 20, // max parallel LLM calls across all scenarios +debounceMs: 0, // optional delay (ms) between scenario starts ``` | Option | Type | Default | Description | @@ -288,6 +289,7 @@ concurrency: 20, // max parallel LLM calls across all scenarios | `conversationsPerScenario` | `number` | `3` | How many independent conversations to run per scenario. More conversations = more statistical confidence, but more LLM cost. | | `maxTurns` | `number` | `8` | Upper bound on user↔agent exchanges. The simulated user can stop earlier if the goal is met. | | `concurrency` | `number` | `20` | Max parallel LLM calls. Controls both simulation and evaluation parallelism. Lower this if you're hitting rate limits. | +| `debounceMs` | `number` | `0` | Minimum delay in milliseconds between scenario starts. Useful for avoiding rate limits when running many scenarios. | Both `conversationsPerScenario` and `maxTurns` can be overridden per-scenario: diff --git a/docs/reference/cli.md b/docs/reference/cli.md index aa62f9a..a7bbafb 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -19,6 +19,7 @@ npx agentest run [options] | `--config ` | `-c` | Path to config file (default: auto-detect) | | `--cwd ` | | Working directory (default: current directory) | | `--scenario ` | `-s` | Filter scenarios by name (case-insensitive substring match) | +| `--file ` | | Filter scenarios by file path (case-insensitive substring match) | | `--verbose` | `-v` | Print full conversation transcripts | | `--watch` | `-w` | Watch mode — re-run on file changes | @@ -39,6 +40,10 @@ npx agentest run --cwd ./packages/my-agent npx agentest run --scenario "booking" npx agentest run --scenario "cancel" +# Filter by file path +npx agentest run --file "auth.sim.ts" +npx agentest run --file "booking" + # Print full conversation transcripts npx agentest run --verbose @@ -48,6 +53,9 @@ npx agentest run -w # Combine flags npx agentest run --watch --verbose --scenario "booking" + +# Combine file and scenario filters +npx agentest run --file "auth" --scenario "login" ``` ### `agentest show-prompts` diff --git a/docs/reference/config-api.md b/docs/reference/config-api.md index b060cde..29b88f5 100644 --- a/docs/reference/config-api.md +++ b/docs/reference/config-api.md @@ -110,6 +110,14 @@ Maximum turns per conversation. Max parallel LLM calls. +### `debounceMs` + +**Type:** `number` + +**Default:** `0` + +Minimum delay in milliseconds between scenario starts. Set this to avoid rate limits when running many scenarios (e.g., `debounceMs: 2000` for a 2-second gap). + ## Evaluation ### `metrics` diff --git a/package.json b/package.json index 3cab35e..7bec7aa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agentesting/agentest", - "version": "0.0.16", + "version": "0.0.17", "description": "Embedded agent simulation & evaluation framework for Node.js/TypeScript", "type": "module", "main": "./dist/index.cjs", diff --git a/src/cli.ts b/src/cli.ts index c8af74b..2d85b03 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -33,6 +33,7 @@ program .option('-c, --config ', 'Path to config file') .option('--cwd ', 'Working directory', process.cwd()) .option('--scenario ', 'Run only scenarios matching this name') + .option('--file ', 'Run only scenarios from files matching this path') .option('--verbose', 'Print full conversation transcripts') .option('-w, --watch', 'Watch for file changes and re-run') .action( @@ -40,6 +41,7 @@ program config?: string cwd: string scenario?: string + file?: string verbose?: boolean watch?: boolean }) => { @@ -47,7 +49,13 @@ program const executeRun = async () => { const config = await loadConfig(configPath, opts.cwd) - const discoveries = await discoverAndLoad(config, opts.cwd) + let discoveries = await discoverAndLoad(config, opts.cwd) + + // Filter by file path if specified + if (opts.file) { + const fileFilter = opts.file.toLowerCase() + discoveries = discoveries.filter((d) => d.file.toLowerCase().includes(fileFilter)) + } // Filter by scenario name if specified if (opts.scenario) {