Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@agentesting/agentest",
"version": "0.0.15",
"version": "0.0.16",
"description": "Embedded agent simulation & evaluation framework for Node.js/TypeScript",
"type": "module",
"main": "./dist/index.cjs",
Expand Down
1 change: 1 addition & 0 deletions src/config/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ export const configSchema = z.object({
maxTurns: z.number().int().positive().default(8),
requestTimeoutMs: z.number().int().min(1000).max(300_000).default(30_000),
concurrency: z.number().int().min(1).max(100).default(20),
debounceMs: z.number().int().min(0).max(300_000).default(0),

metrics: z
.array(
Expand Down
20 changes: 17 additions & 3 deletions src/runner/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,13 +60,24 @@ export class Runner {
private config: AgentestConfig
private reporters: Reporter[]
private limit: ReturnType<typeof pLimit>
private lastScenarioStartTime = 0

constructor(config: AgentestConfig, reporters: Reporter[]) {
this.config = config
this.reporters = reporters
this.limit = pLimit(config.concurrency)
}

private async debounce(): Promise<void> {
if (this.config.debounceMs <= 0) return
const elapsed = Date.now() - this.lastScenarioStartTime
const remaining = this.config.debounceMs - elapsed
if (remaining > 0) {
await new Promise((resolve) => setTimeout(resolve, remaining))
}
this.lastScenarioStartTime = Date.now()
}

private async createInfrastructure() {
const llm = await createProvider(
this.config.provider,
Expand Down Expand Up @@ -105,9 +116,10 @@ export class Runner {
// Run all scenarios in parallel (bounded by concurrency)
const scenarioResults = await Promise.all(
allScenarios.map((scenario) =>
this.limit(() =>
this.runScenario(scenario, simulator, evaluator, trajectoryMatcher, errorDetection),
),
this.limit(async () => {
await this.debounce()
return this.runScenario(scenario, simulator, evaluator, trajectoryMatcher, errorDetection)
}),
),
)

Expand Down Expand Up @@ -210,6 +222,8 @@ export class Runner {
}

for (const scenario of allScenarios) {
await this.debounce()

// Run all agents in parallel for this scenario
const agentResults = await Promise.all(
agentNames.map((agentName) =>
Expand Down
Loading