diff --git a/package.json b/package.json index f235cd4..3cab35e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agentesting/agentest", - "version": "0.0.15", + "version": "0.0.16", "description": "Embedded agent simulation & evaluation framework for Node.js/TypeScript", "type": "module", "main": "./dist/index.cjs", diff --git a/src/config/schema.ts b/src/config/schema.ts index 49cd169..90ac74e 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -126,6 +126,7 @@ export const configSchema = z.object({ maxTurns: z.number().int().positive().default(8), requestTimeoutMs: z.number().int().min(1000).max(300_000).default(30_000), concurrency: z.number().int().min(1).max(100).default(20), + debounceMs: z.number().int().min(0).max(300_000).default(0), metrics: z .array( diff --git a/src/runner/runner.ts b/src/runner/runner.ts index 60cdc78..5dc80e4 100644 --- a/src/runner/runner.ts +++ b/src/runner/runner.ts @@ -60,6 +60,7 @@ export class Runner { private config: AgentestConfig private reporters: Reporter[] private limit: ReturnType + private lastScenarioStartTime = 0 constructor(config: AgentestConfig, reporters: Reporter[]) { this.config = config @@ -67,6 +68,16 @@ export class Runner { this.limit = pLimit(config.concurrency) } + private async debounce(): Promise { + if (this.config.debounceMs <= 0) return + const elapsed = Date.now() - this.lastScenarioStartTime + const remaining = this.config.debounceMs - elapsed + if (remaining > 0) { + await new Promise((resolve) => setTimeout(resolve, remaining)) + } + this.lastScenarioStartTime = Date.now() + } + private async createInfrastructure() { const llm = await createProvider( this.config.provider, @@ -105,9 +116,10 @@ export class Runner { // Run all scenarios in parallel (bounded by concurrency) const scenarioResults = await Promise.all( allScenarios.map((scenario) => - this.limit(() => - this.runScenario(scenario, simulator, evaluator, trajectoryMatcher, errorDetection), - ), + this.limit(async () => { + await this.debounce() + return this.runScenario(scenario, simulator, evaluator, trajectoryMatcher, errorDetection) + }), ), ) @@ -210,6 +222,8 @@ export class Runner { } for (const scenario of allScenarios) { + await this.debounce() + // Run all agents in parallel for this scenario const agentResults = await Promise.all( agentNames.map((agentName) =>