From 8f9a323f56e34dc8791cabee64fe1f98341c2389 Mon Sep 17 00:00:00 2001 From: Yun Hong <155888058+16yunH@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:29:05 +0800 Subject: [PATCH 1/2] examples: add eval optimization regression loop Add a no-key Evaluation and Optimization pipeline that baselines train and validation data, captures every GEPA proposal, independently re-evaluates candidates, applies regression and resource gates, and emits auditable JSON and Markdown reports. Include deterministic replay data, prompt candidates, failure attribution, paired bootstrap confidence intervals, Pareto selection, data-leakage checks, sample output, and regression tests. Fixes #91 RELEASE NOTES: Add an auditable offline Evaluation and Optimization regression loop example. --- .../eval_optimize_loop/.gitignore | 3 + .../optimization/eval_optimize_loop/DESIGN.md | 3 + .../optimization/eval_optimize_loop/README.md | 115 + .../eval_optimize_loop/__init__.py | 8 + .../eval_optimize_loop/agent/__init__.py | 8 + .../agent/prompts/candidates/ineffective.md | 3 + .../agent/prompts/candidates/overfit.md | 3 + .../agent/prompts/candidates/robust.md | 3 + .../agent/prompts/system.md | 3 + .../data/train.evalset.json | 189 + .../eval_optimize_loop/data/val.evalset.json | 189 + .../optimization/eval_optimize_loop/gate.json | 13 + .../eval_optimize_loop/loop/__init__.py | 18 + .../eval_optimize_loop/loop/analysis.py | 538 +++ .../eval_optimize_loop/loop/models.py | 289 ++ .../eval_optimize_loop/loop/offline.py | 294 ++ .../eval_optimize_loop/loop/pipeline.py | 284 ++ .../eval_optimize_loop/loop/reporting.py | 152 + .../eval_optimize_loop/loop/trace.py | 543 +++ .../eval_optimize_loop/optimizer.json | 81 + .../eval_optimize_loop/pipeline.json | 29 + .../regression.metrics.json | 81 + .../eval_optimize_loop/run_pipeline.py | 77 + .../sample_output/optimization_report.json | 3401 +++++++++++++++++ .../sample_output/optimization_report.md | 123 + .../test_eval_optimize_loop_example.py | 705 ++++ 26 files changed, 7155 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/.gitignore create mode 100644 examples/optimization/eval_optimize_loop/DESIGN.md create mode 100644 examples/optimization/eval_optimize_loop/README.md create mode 100644 examples/optimization/eval_optimize_loop/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/agent/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/agent/prompts/candidates/ineffective.md create mode 100644 examples/optimization/eval_optimize_loop/agent/prompts/candidates/overfit.md create mode 100644 examples/optimization/eval_optimize_loop/agent/prompts/candidates/robust.md create mode 100644 examples/optimization/eval_optimize_loop/agent/prompts/system.md create mode 100644 examples/optimization/eval_optimize_loop/data/train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/val.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/gate.json create mode 100644 examples/optimization/eval_optimize_loop/loop/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/loop/analysis.py create mode 100644 examples/optimization/eval_optimize_loop/loop/models.py create mode 100644 examples/optimization/eval_optimize_loop/loop/offline.py create mode 100644 examples/optimization/eval_optimize_loop/loop/pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/loop/reporting.py create mode 100644 examples/optimization/eval_optimize_loop/loop/trace.py create mode 100644 examples/optimization/eval_optimize_loop/optimizer.json create mode 100644 examples/optimization/eval_optimize_loop/pipeline.json create mode 100644 examples/optimization/eval_optimize_loop/regression.metrics.json create mode 100644 examples/optimization/eval_optimize_loop/run_pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.md create mode 100644 tests/evaluation/test_eval_optimize_loop_example.py diff --git a/examples/optimization/eval_optimize_loop/.gitignore b/examples/optimization/eval_optimize_loop/.gitignore new file mode 100644 index 00000000..6d0ca127 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/.gitignore @@ -0,0 +1,3 @@ +runs/ +sample_output/optimizer/ +sample_output/traces/ diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 00000000..8e5c0ed6 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,3 @@ +# 设计说明（约 400 字） + +闭环以 `EvalOptimizePipeline.run()` 作为唯一外部接口，把数据校验、基线评估、提示词优化、候选复评、差异计算、门禁和报告隐藏在同一深模块内。优化阶段直接复用 `AgentOptimizer`、`TargetPrompt` 与真实 GEPA；无密钥模式仅通过 `ModelRegistry` 注入离线 agent、judge 和 reflector，因而仍会经过 `LlmAgent`、内置 rubric 解析及优化器回调，不依赖 monkeypatch。外层将每个候选运行物化为 trace，再用 `AgentEvaluator` 独立评测 train 与 validation，避免采信优化器自己的聚合分数。候选集合按完整提示词哈希去重，即使优化器内部提前拒绝某次提案，外层仍会捕获并独立复评，防止遗漏验证回退。逐 case 差异区分新增通过、新增失败、升分、降分和不变；归因按工具执行、参数、格式、知识召回、rubric、最终回答的根因优先级输出证据。SDK 聚合器偶尔不保留逐 rubric 详情，格式归因依次采用详情、回放证据和请求中明确的 JSON、单行或 Markdown 约束，否定式要求不触发检查；judge 无有效 verdict 时单列 evaluation_error，避免把系统错误或内容失败误报为格式失败。Gate 对验证增益、配对 bootstrap 下界、新增 hard fail、关键 case、过拟合及资源预算做可配置 AND，并先过滤不安全候选，再计算质量、token、P95 时延 Pareto。运行前检查跨集合重复、近重复和答案泄漏；运行后保存完整 prompt、seed、哈希、耗时、成本、trace 与 JSON/Markdown 报告。默认始终恢复源 prompt，只有显式允许且 Gate 全通过才原子回写。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..213a0fa1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,115 @@ +# Evaluation + Optimization 自动回归闭环 + +本示例把一次 prompt 优化从“分数变高了”升级为可审计的发布决策：先在 train/validation 上建立 baseline，再让真实 `AgentOptimizer` 与 GEPA 生成候选，随后对**每个去重候选**做独立 trace 复评、逐 case diff、失败归因和 Gate 判定，最后同时输出 JSON 与 Markdown 报告。默认模式不读取任何 API Key，也不访问网络。 + +## 一键运行 + +在仓库根目录执行： + +```bash +pip install -e ".[eval,optimize]" +python examples/optimization/eval_optimize_loop/run_pipeline.py +``` + +无需设置 `OPENAI_API_KEY`、`TRPC_AGENT_API_KEY` 或其他模型凭据。指定输出目录： + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --output-dir /tmp/eval-optimize-loop +``` + +命令正常完成时输出 `decision=accepted` 与报告路径。默认不会覆盖源 prompt；只有显式传入 `--apply-if-accepted` 且所有必选 Gate 均通过，才会原子回写最终候选。 + +## 为什么不是“伪造优化” + +离线 provider 通过框架公开的 `ModelRegistry` 注册三个模型名，并真实经过 SDK 的生产调用链： + +```text +AgentOptimizer.optimize + -> GepaReflectiveOptimizer + -> gepa.optimize + -> AgentEvaluator + -> LlmAgent(offline/agent) + -> 内置 LLMRubricResponseEvaluator(offline/judge) + +reflection + -> _OptimizeModelCallable + -> LlmAgent(offline/reflector) +``` + +离线模型只替换远程推理这一处依赖；候选写入/恢复、GEPA 采样和筛选、rubric JSON 解析、评估聚合、回调及优化器 artifact 全部使用真实框架实现，没有 monkeypatch。外层回归会把每种 prompt 的确定性运行结果物化为 `actual_conversation`，再用 `AgentEvaluator` 的 trace-only 模式独立评分。 + +## 示例数据与三类候选 + +`data/train.evalset.json` 与 `data/val.evalset.json` 各有 3 个 case。它们不在实现代码中硬编码 case id，离线运行行为完全来自每个 case 的 `session_input.state.variant_traces`。 + +| 候选 | Train | Validation | 预期 Gate | +| --- | --- | --- | --- | +| `ineffective` | 无提升 | 无提升 | 拒绝：validation 增益不足 | +| `overfit` | 3/3 通过 | 关键 JSON case 从通过变失败 | 拒绝：hard fail、关键 case 回退、过拟合 | +| `robust` | 3/3 通过 | 3/3 通过 | 接受 | + +六个 case 覆盖：最终回答不匹配、工具执行错误、工具参数错误、LLM rubric 失败、知识召回不足和格式失败。归因器从 actual/expected 工具轨迹、工具响应和 evaluator metric 中推导原因，不读取预埋的 `failure_category` 标签。格式归因优先使用逐 rubric 详情，其次使用 replay 证据，最后只解析请求中明确、非否定的 JSON、单行或 Markdown 约束；evaluator 没有产生有效 verdict 时单列 `evaluation_error`，不会伪装成语义失败。 + +## 输入 + +| 文件 | 用途 | +| --- | --- | +| `pipeline.json` | 路径清单、seed、bootstrap 次数和是否回写 | +| `optimizer.json` | 可直接被 SDK `load_optimize_config()` 校验的 GEPA 配置 | +| `regression.metrics.json` | 外层 trace 复评的回答、轨迹、rubric、知识召回指标 | +| `gate.json` | 增益、置信下界、hard fail、关键 case、过拟合和预算规则 | +| `agent/prompts/system.md` | baseline `TargetPrompt` 源文件 | +| `agent/prompts/candidates/*.md` | 离线 reflection model 的确定性候选 | +| `data/*.evalset.json` | train / validation case 与 replay trace | + +## Gate 语义 + +每一项都在报告中记录 `passed / required / actual / expected / reason`。必选项做 AND： + +1. `AgentOptimizer` 成功结束； +2. validation pass-rate 增益达到 `min_validation_gain`； +3. paired bootstrap 区间下界达到配置值； +4. 不新增 hard failure； +5. key case 不降分； +6. train 提升而 validation 下降时判定 overfit 并拒绝；趋势以 pass-rate 为主，持平时再比较 average score； +7. metric calls、token、耗时和成本均在预算内。 + +`forbid_new_hard_failures`、`key_cases_no_regression`、`reject_overfitting` 可以显式设为 `false`；报告仍保留检查结果，但 `required=false`，不会参与最终 AND。候选成本从每条 replay trace 的 `usage.cost`/`usage.cost_usd` 汇总并乘以 `num_runs`；任一值缺失时标记为 `unavailable`，成本 Gate fail-closed。仓库自带的离线 trace、judge 和 optimizer 均为本地确定性计算，因此样例成本被明确记录为“已测量的 0”，而不是把未知误写成 0。 + +## 统计与数据污染防护 + +- baseline 与 candidate 在同一 case 上配对，使用固定 seed 的 2,000 次 bootstrap，报告 validation pass-rate delta 的 95% 区间。 +- 先过安全 Gate，再在合格候选中按 validation 质量、token 与 P95 latency 标记 Pareto 前沿。 +- 运行前检查重复 id、train/validation 精确重复、去空白后的重复、相似度 ≥0.92 的近重复，以及 baseline/candidate prompt 中直接出现 validation reference answer；命中即 fail-closed。 +- baseline、每个候选和每个 split 的 trace 均保存到 artifact 目录，可由 `AgentEvaluator` 重新播放。 + +## 输出 + +```text +runs// +├── optimization_report.json 机器可读的完整闭环报告 +├── optimization_report.md 人类可读决策报告 +├── optimizer/ AgentOptimizer 原始 result/round artifacts +└── traces/ + ├── baseline/{train,validation}.trace.evalset.json + ├── ineffective/{train,validation}.trace.evalset.json + ├── overfit/{train,validation}.trace.evalset.json + └── robust/{train,validation}.trace.evalset.json +``` + +JSON 顶层包含 `baseline`、`candidates`、选中 `candidate`、`delta`、`gate`、`failure_attribution`、`optimizer`、`data_quality` 与 `audit`。每个候选保存完整 prompt、SHA-256、seed、来源轮次、逐 case delta、失败证据、metric/judge 调用数、token、P95 latency、耗时与成本。 + +仓库内的固定示例报告见 `sample_output/`。 + +## 验证 + +```bash +pytest -q tests/evaluation/test_eval_optimize_loop_example.py +``` + +测试覆盖无 Key 全链路、全部候选独立复评、六类失败归因与反误判、score-only 过拟合、paired bootstrap、Pareto、近重复数据拒绝、可配置保护项、成本 fail-closed 和逐候选资源审计。离线完整流程通常在数秒内完成，远低于 3 分钟验收上限。 + +## 接入真实业务 + +保留 `EvalOptimizePipeline.run()` 这个外部 interface，将 `offline.py` 的三个 adapter 替换为业务实现：`call_agent` 驱动真实 Agent，reflection/judge 的 `provider_name` 改为实际 provider，trace 物化器读取业务运行日志。`optimizer.json`、`TargetPrompt`、逐候选复评、diff、Gate 和报告模型无需改变。生产环境建议每次使用唯一 output directory；离线 registry 使用进程级固定 replay 配置，不支持在同一 Python 进程中并发启动两个 pipeline。 diff --git a/examples/optimization/eval_optimize_loop/__init__.py b/examples/optimization/eval_optimize_loop/__init__.py new file mode 100644 index 00000000..20778b26 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/__init__.py @@ -0,0 +1,8 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Evaluation + optimization regression-loop example.""" diff --git a/examples/optimization/eval_optimize_loop/agent/__init__.py b/examples/optimization/eval_optimize_loop/agent/__init__.py new file mode 100644 index 00000000..e8667a22 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/__init__.py @@ -0,0 +1,8 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Prompt assets for the offline support-agent scenario.""" diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/candidates/ineffective.md b/examples/optimization/eval_optimize_loop/agent/prompts/candidates/ineffective.md new file mode 100644 index 00000000..95d1b600 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/candidates/ineffective.md @@ -0,0 +1,3 @@ +[variant: ineffective] + +You are a helpful customer-support agent. Be friendly and concise. diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/candidates/overfit.md b/examples/optimization/eval_optimize_loop/agent/prompts/candidates/overfit.md new file mode 100644 index 00000000..710b7085 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/candidates/overfit.md @@ -0,0 +1,3 @@ +[variant: overfit] + +You are a customer-support agent tuned to the three training examples. Memorize their answers and prefer them over general rules. diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/candidates/robust.md b/examples/optimization/eval_optimize_loop/agent/prompts/candidates/robust.md new file mode 100644 index 00000000..3ed8f19b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/candidates/robust.md @@ -0,0 +1,3 @@ +[variant: robust] + +You are a reliable customer-support agent. Select the correct tool, preserve exact identifiers and arguments, ground the answer in retrieved evidence, follow the requested format, and include every required step in order. diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/system.md b/examples/optimization/eval_optimize_loop/agent/prompts/system.md new file mode 100644 index 00000000..c83dce09 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/system.md @@ -0,0 +1,3 @@ +[variant: baseline] + +You are a customer-support agent. Give a short answer from the available evidence. diff --git a/examples/optimization/eval_optimize_loop/data/train.evalset.json b/examples/optimization/eval_optimize_loop/data/train.evalset.json new file mode 100644 index 00000000..20d917a9 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/train.evalset.json @@ -0,0 +1,189 @@ +{ + "eval_set_id": "support_regression_train", + "name": "离线客服回归训练集", + "description": "三个训练 case，分别暴露最终回答、工具选择和参数错误。", + "eval_cases": [ + { + "eval_id": "train_final_response_mismatch", + "conversation": [ + { + "invocation_id": "expected-1", + "user_content": { + "role": "user", + "parts": [{"text": "请确认售后工单 T-100 的状态。"}] + }, + "final_response": { + "role": "model", + "parts": [{"text": "工单 T-100 状态：已完成。"}] + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "offline", + "state": { + "key_case": false, + "hard_fail": false, + "variant_traces": { + "baseline": { + "final_response": {"role": "model", "parts": [{"text": "工单 T-100 状态：处理中。"}]}, + "intermediate_data": {"tool_uses": [], "tool_responses": []}, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 36, "output_tokens": 14, "cost": 0.0, "latency_ms": 4} + }, + "ineffective": { + "final_response": {"role": "model", "parts": [{"text": "工单 T-100 状态：处理中。"}]}, + "intermediate_data": {"tool_uses": [], "tool_responses": []}, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 38, "output_tokens": 14, "cost": 0.0, "latency_ms": 4} + }, + "overfit": { + "final_response": {"role": "model", "parts": [{"text": "工单 T-100 状态：已完成。"}]}, + "intermediate_data": {"tool_uses": [], "tool_responses": []}, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 42, "output_tokens": 14, "cost": 0.0, "latency_ms": 5} + }, + "robust": { + "final_response": {"role": "model", "parts": [{"text": "工单 T-100 状态：已完成。"}]}, + "intermediate_data": {"tool_uses": [], "tool_responses": []}, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 48, "output_tokens": 14, "cost": 0.0, "latency_ms": 5} + } + } + } + } + }, + { + "eval_id": "train_tool_call_error", + "conversation": [ + { + "invocation_id": "expected-2", + "user_content": { + "role": "user", + "parts": [{"text": "查询订单 O-200 的物流状态。"}] + }, + "final_response": { + "role": "model", + "parts": [{"text": "订单 O-200 状态：已发货。"}] + }, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-200"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-200", "status": "已发货"}}] + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "offline", + "state": { + "key_case": false, + "hard_fail": true, + "variant_traces": { + "baseline": { + "final_response": {"role": "model", "parts": [{"text": "无法查询订单状态。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-200"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"error": "backend unavailable"}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 40, "output_tokens": 12, "cost": 0.0, "latency_ms": 7} + }, + "ineffective": { + "final_response": {"role": "model", "parts": [{"text": "无法查询订单状态。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-200"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"error": "backend unavailable"}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 42, "output_tokens": 12, "cost": 0.0, "latency_ms": 7} + }, + "overfit": { + "final_response": {"role": "model", "parts": [{"text": "订单 O-200 状态：已发货。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-200"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-200", "status": "已发货"}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 46, "output_tokens": 16, "cost": 0.0, "latency_ms": 6} + }, + "robust": { + "final_response": {"role": "model", "parts": [{"text": "订单 O-200 状态：已发货。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-200"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-200", "status": "已发货"}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 52, "output_tokens": 16, "cost": 0.0, "latency_ms": 6} + } + } + } + } + }, + { + "eval_id": "train_parameter_error", + "conversation": [ + { + "invocation_id": "expected-3", + "user_content": { + "role": "user", + "parts": [{"text": "查询订单 O-300 的物流状态。"}] + }, + "final_response": { + "role": "model", + "parts": [{"text": "订单 O-300 状态：处理中。"}] + }, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-300"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-300", "status": "处理中"}}] + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "offline", + "state": { + "key_case": false, + "hard_fail": true, + "variant_traces": { + "baseline": { + "final_response": {"role": "model", "parts": [{"text": "订单 O-399 状态：已取消。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-399"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-399", "status": "已取消"}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 40, "output_tokens": 16, "cost": 0.0, "latency_ms": 5} + }, + "ineffective": { + "final_response": {"role": "model", "parts": [{"text": "订单 O-399 状态：已取消。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-399"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-399", "status": "已取消"}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 42, "output_tokens": 16, "cost": 0.0, "latency_ms": 5} + }, + "overfit": { + "final_response": {"role": "model", "parts": [{"text": "订单 O-300 状态：处理中。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-300"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-300", "status": "处理中"}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 46, "output_tokens": 16, "cost": 0.0, "latency_ms": 5} + }, + "robust": { + "final_response": {"role": "model", "parts": [{"text": "订单 O-300 状态：处理中。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-300"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-300", "status": "处理中"}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 52, "output_tokens": 16, "cost": 0.0, "latency_ms": 5} + } + } + } + } + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/data/val.evalset.json b/examples/optimization/eval_optimize_loop/data/val.evalset.json new file mode 100644 index 00000000..b98c1532 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/val.evalset.json @@ -0,0 +1,189 @@ +{ + "eval_set_id": "support_regression_validation", + "name": "离线客服回归验证集", + "description": "三个验证 case，覆盖 rubric、知识召回和关键格式回退。", + "eval_cases": [ + { + "eval_id": "val_llm_rubric_failure", + "conversation": [ + { + "invocation_id": "expected-4", + "user_content": { + "role": "user", + "parts": [{"text": "客户申请退款时，请说明客服必须先做什么，再做什么。"}] + }, + "final_response": { + "role": "model", + "parts": [{"text": "退款处理流程："}] + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "offline", + "state": { + "key_case": false, + "hard_fail": false, + "variant_traces": { + "baseline": { + "final_response": {"role": "model", "parts": [{"text": "退款处理流程：请告知客户耐心等待。"}]}, + "intermediate_data": {"tool_uses": [], "tool_responses": []}, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": false}, + "usage": {"input_tokens": 44, "output_tokens": 18, "cost": 0.0, "latency_ms": 5} + }, + "ineffective": { + "final_response": {"role": "model", "parts": [{"text": "退款处理流程：请告知客户耐心等待。"}]}, + "intermediate_data": {"tool_uses": [], "tool_responses": []}, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": false}, + "usage": {"input_tokens": 46, "output_tokens": 18, "cost": 0.0, "latency_ms": 5} + }, + "overfit": { + "final_response": {"role": "model", "parts": [{"text": "退款处理流程：请告知客户耐心等待。"}]}, + "intermediate_data": {"tool_uses": [], "tool_responses": []}, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": false}, + "usage": {"input_tokens": 50, "output_tokens": 18, "cost": 0.0, "latency_ms": 5} + }, + "robust": { + "final_response": {"role": "model", "parts": [{"text": "退款处理流程：先核验订单与付款状态，再告知退款时效。"}]}, + "intermediate_data": {"tool_uses": [], "tool_responses": []}, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 56, "output_tokens": 24, "cost": 0.0, "latency_ms": 6} + } + } + } + } + }, + { + "eval_id": "val_knowledge_recall_insufficiency", + "conversation": [ + { + "invocation_id": "expected-5", + "user_content": { + "role": "user", + "parts": [{"text": "根据知识库回答：会员退款到账需要多久？"}] + }, + "final_response": { + "role": "model", + "parts": [{"text": "会员退款到账："}] + }, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "search_policy", "args": {"query": "会员退款到账时效", "top_k": 2}}], + "tool_responses": [{"id": "call-1", "name": "search_policy", "response": {"documents": [{"id": "refund-sla", "text": "会员退款原路退回，3-5 个工作日到账。"}]}}] + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "offline", + "state": { + "key_case": false, + "hard_fail": false, + "variant_traces": { + "baseline": { + "final_response": {"role": "model", "parts": [{"text": "会员退款到账：通常当天到账。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "search_policy", "args": {"query": "会员退款到账时效", "top_k": 2}}], + "tool_responses": [{"id": "call-1", "name": "search_policy", "response": {"documents": [{"id": "return-window", "text": "商品支持 7 天退换。"}]}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": false, "llm_rubric_pass": false}, + "usage": {"input_tokens": 48, "output_tokens": 18, "cost": 0.0, "latency_ms": 8} + }, + "ineffective": { + "final_response": {"role": "model", "parts": [{"text": "会员退款到账：通常当天到账。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "search_policy", "args": {"query": "会员退款到账时效", "top_k": 2}}], + "tool_responses": [{"id": "call-1", "name": "search_policy", "response": {"documents": [{"id": "return-window", "text": "商品支持 7 天退换。"}]}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": false, "llm_rubric_pass": false}, + "usage": {"input_tokens": 50, "output_tokens": 18, "cost": 0.0, "latency_ms": 8} + }, + "overfit": { + "final_response": {"role": "model", "parts": [{"text": "会员退款到账：通常当天到账。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "search_policy", "args": {"query": "会员退款到账时效", "top_k": 2}}], + "tool_responses": [{"id": "call-1", "name": "search_policy", "response": {"documents": [{"id": "return-window", "text": "商品支持 7 天退换。"}]}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": false, "llm_rubric_pass": false}, + "usage": {"input_tokens": 54, "output_tokens": 18, "cost": 0.0, "latency_ms": 8} + }, + "robust": { + "final_response": {"role": "model", "parts": [{"text": "会员退款到账：原路退回，3-5 个工作日到账。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "search_policy", "args": {"query": "会员退款到账时效", "top_k": 2}}], + "tool_responses": [{"id": "call-1", "name": "search_policy", "response": {"documents": [{"id": "refund-sla", "text": "会员退款原路退回，3-5 个工作日到账。"}]}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 60, "output_tokens": 24, "cost": 0.0, "latency_ms": 8} + } + } + } + } + }, + { + "eval_id": "val_format_key_case", + "conversation": [ + { + "invocation_id": "expected-6", + "user_content": { + "role": "user", + "parts": [{"text": "查询订单 O-400，只返回 JSON，字段为 order_id 和 status。"}] + }, + "final_response": { + "role": "model", + "parts": [{"text": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}"}] + }, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-400"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-400", "status": "已签收"}}] + } + } + ], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "offline", + "state": { + "key_case": true, + "hard_fail": true, + "variant_traces": { + "baseline": { + "final_response": {"role": "model", "parts": [{"text": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-400"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-400", "status": "已签收"}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 50, "output_tokens": 20, "cost": 0.0, "latency_ms": 6} + }, + "ineffective": { + "final_response": {"role": "model", "parts": [{"text": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-400"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-400", "status": "已签收"}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 52, "output_tokens": 20, "cost": 0.0, "latency_ms": 6} + }, + "overfit": { + "final_response": {"role": "model", "parts": [{"text": "订单 O-400 状态：已签收。"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-400"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-400", "status": "已签收"}}] + }, + "signals": {"format_pass": false, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 56, "output_tokens": 16, "cost": 0.0, "latency_ms": 6} + }, + "robust": { + "final_response": {"role": "model", "parts": [{"text": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}"}]}, + "intermediate_data": { + "tool_uses": [{"id": "call-1", "name": "lookup_order", "args": {"order_id": "O-400"}}], + "tool_responses": [{"id": "call-1", "name": "lookup_order", "response": {"order_id": "O-400", "status": "已签收"}}] + }, + "signals": {"format_pass": true, "knowledge_recall_pass": true, "llm_rubric_pass": true}, + "usage": {"input_tokens": 62, "output_tokens": 20, "cost": 0.0, "latency_ms": 6} + } + } + } + } + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/gate.json b/examples/optimization/eval_optimize_loop/gate.json new file mode 100644 index 00000000..40017c97 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/gate.json @@ -0,0 +1,13 @@ +{ + "min_validation_gain": 0.5, + "min_validation_gain_ci_lower_bound": 0.0, + "forbid_new_hard_failures": true, + "key_cases_no_regression": true, + "reject_overfitting": true, + "budget": { + "max_metric_calls": 80, + "max_total_tokens": 200000, + "max_duration_seconds": 180, + "max_cost_usd": 0.0 + } +} diff --git a/examples/optimization/eval_optimize_loop/loop/__init__.py b/examples/optimization/eval_optimize_loop/loop/__init__.py new file mode 100644 index 00000000..b8f3366e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/loop/__init__.py @@ -0,0 +1,18 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Public interface for the evaluation/optimization regression loop.""" + +from .models import OptimizationReport +from .models import PipelineSpec +from .pipeline import EvalOptimizePipeline + +__all__ = [ + "EvalOptimizePipeline", + "OptimizationReport", + "PipelineSpec", +] diff --git a/examples/optimization/eval_optimize_loop/loop/analysis.py b/examples/optimization/eval_optimize_loop/loop/analysis.py new file mode 100644 index 00000000..91217291 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/loop/analysis.py @@ -0,0 +1,538 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Pure regression analysis and promotion policy for the closed loop.""" + +from __future__ import annotations + +import hashlib +import json +import math +import random +import re +from collections import Counter +from difflib import SequenceMatcher +from pathlib import Path +from typing import Any + +from trpc_agent_sdk.evaluation import EvalCase +from trpc_agent_sdk.evaluation import EvalConfig +from trpc_agent_sdk.evaluation import EvalSet + +from .models import BaselineEvaluation +from .models import CandidateDelta +from .models import CandidateEvaluation +from .models import CaseDelta +from .models import CaseEvaluation +from .models import DataQualityAudit +from .models import FailureAttributionSummary +from .models import GateCheck +from .models import GateDecision +from .models import PairedConfidenceInterval +from .models import ResourceUsage +from .models import SplitDelta +from .models import SplitEvaluation + + +def _sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _text(content: Any) -> str: + if content is None: + return "" + return "\n".join(str(getattr(part, "text", "") or "") for part in (content.parts or [])).strip() + + +def _normalize(value: str) -> str: + return re.sub(r"\s+", "", value).casefold() + + +class RegressionAnalyzer: + """Compute comparable deltas, resource evidence, and promotion decisions.""" + + def __init__( + self, + *, + seed: int, + bootstrap_samples: int, + confidence_level: float, + ) -> None: + self._seed = seed + self._bootstrap_samples = bootstrap_samples + self._confidence_level = confidence_level + + def validate_data_quality( + self, + train_set: EvalSet, + validation_set: EvalSet, + *, + train_path: Path, + validation_path: Path, + prompt_text: str, + ) -> DataQualityAudit: + """Reject split contamination and validation-answer leakage.""" + if train_path == validation_path: + raise ValueError("train and validation paths must differ") + if len(train_set.eval_cases) < 3 or len(validation_set.eval_cases) < 3: + raise ValueError("train and validation sets must each contain at least 3 cases") + + all_ids = [case.eval_id for case in train_set.eval_cases + validation_set.eval_cases] + duplicate_ids = sorted(case_id for case_id, count in Counter(all_ids).items() if count > 1) + train_fingerprints = {self._case_fingerprint(case): case.eval_id for case in train_set.eval_cases} + validation_fingerprints = {self._case_fingerprint(case): case.eval_id for case in validation_set.eval_cases} + overlap = sorted(set(train_fingerprints) & set(validation_fingerprints)) + cross_split = [f"{train_fingerprints[item]} == {validation_fingerprints[item]}" for item in overlap] + + near_cross_split: list[str] = [] + for train_case in train_set.eval_cases: + train_text = self._case_normalized_content(train_case) + for validation_case in validation_set.eval_cases: + validation_text = self._case_normalized_content(validation_case) + if train_text == validation_text: + continue + similarity = SequenceMatcher(None, train_text, validation_text).ratio() + if min(len(train_text), len(validation_text)) >= 24 and similarity >= 0.92: + near_cross_split.append(f"{train_case.eval_id} ~= {validation_case.eval_id} ({similarity:.3f})") + + prompt_normalized = _normalize(prompt_text) + leakage: list[str] = [] + for case in validation_set.eval_cases: + expected = self._expected_response(case) + normalized = _normalize(expected) + if len(normalized) >= 12 and normalized in prompt_normalized: + leakage.append(case.eval_id) + if duplicate_ids or cross_split or near_cross_split or leakage: + raise ValueError("data quality check failed: " + f"duplicate_ids={duplicate_ids}, cross_split={cross_split}, " + f"near_cross_split={near_cross_split}, prompt_leakage={leakage}") + return DataQualityAudit( + passed=True, + train_cases=len(train_set.eval_cases), + validation_cases=len(validation_set.eval_cases), + duplicate_ids=duplicate_ids, + cross_split_duplicates=cross_split, + near_cross_split_duplicates=near_cross_split, + prompt_leakage_matches=leakage, + ) + + @staticmethod + def _case_fingerprint(case: EvalCase) -> str: + return _sha256_text(RegressionAnalyzer._case_normalized_content(case)) + + @staticmethod + def _case_normalized_content(case: EvalCase) -> str: + conversation = case.conversation or [] + payload = "\n".join(f"{_text(invocation.user_content)}\n{_text(invocation.final_response)}" + for invocation in conversation) + return _normalize(payload) + + @staticmethod + def _expected_response(case: EvalCase) -> str: + conversation = case.conversation or [] + return _text(conversation[-1].final_response) if conversation else "" + + def diff(self, baseline: SplitEvaluation, candidate: SplitEvaluation) -> SplitDelta: + """Build a complete, paired per-case delta for one split.""" + baseline_by_id = {case.case_id: case for case in baseline.cases} + candidate_by_id = {case.case_id: case for case in candidate.cases} + if set(baseline_by_id) != set(candidate_by_id): + raise ValueError("baseline and candidate case sets differ; refusing partial delta") + buckets: dict[str, list[str]] = { + "newly_passed": [], + "newly_failed": [], + "score_improved": [], + "score_regressed": [], + "unchanged": [], + } + cases: list[CaseDelta] = [] + for case_id in sorted(baseline_by_id): + before = baseline_by_id[case_id] + after = candidate_by_id[case_id] + score_delta = after.score - before.score + if not before.passed and after.passed: + status = "newly_passed" + elif before.passed and not after.passed: + status = "newly_failed" + elif score_delta > 1e-12: + status = "score_improved" + elif score_delta < -1e-12: + status = "score_regressed" + else: + status = "unchanged" + buckets[status].append(case_id) + cases.append( + CaseDelta( + case_id=case_id, + status=status, + baseline_passed=before.passed, + candidate_passed=after.passed, + baseline_score=before.score, + candidate_score=after.score, + score_delta=score_delta, + )) + return SplitDelta( + split=baseline.split, + pass_rate_delta=candidate.pass_rate - baseline.pass_rate, + average_score_delta=candidate.average_score - baseline.average_score, + paired_pass_rate_ci=self._paired_bootstrap([ + float(candidate_by_id[case_id].passed) - float(baseline_by_id[case_id].passed) + for case_id in sorted(baseline_by_id) + ]), + cases=cases, + **buckets, + ) + + def _paired_bootstrap(self, paired_deltas: list[float]) -> PairedConfidenceInterval: + if not paired_deltas: + raise ValueError("paired bootstrap requires at least one case") + rng = random.Random(self._seed) + sample_size = len(paired_deltas) + estimates = [] + for _ in range(self._bootstrap_samples): + estimate = sum(paired_deltas[rng.randrange(sample_size)] for _ in range(sample_size)) / sample_size + estimates.append(estimate) + estimates.sort() + tail = (1.0 - self._confidence_level) / 2.0 + lower_index = max(0, min(len(estimates) - 1, int(tail * len(estimates)))) + upper_index = max( + 0, + min(len(estimates) - 1, + int((1.0 - tail) * len(estimates)) - 1), + ) + return PairedConfidenceInterval( + point_estimate=sum(paired_deltas) / sample_size, + lower=estimates[lower_index], + upper=estimates[upper_index], + confidence_level=self._confidence_level, + bootstrap_samples=self._bootstrap_samples, + seed=self._seed, + ) + + @staticmethod + def optimizer_resources(result: Any) -> ResourceUsage: + """Normalize AgentOptimizer accounting into the shared audit model.""" + usage = result.total_token_usage or {} + metric_calls = int((result.extras or {}).get("total_metric_calls", 0)) + return ResourceUsage( + metric_calls=metric_calls, + reflection_calls=result.total_reflection_lm_calls, + judge_calls=None, + prompt_tokens=int(usage.get("prompt", 0)), + completion_tokens=int(usage.get("completion", 0)), + total_tokens=int(usage.get("total", 0)), + cost_usd=0.0, + duration_seconds=float(result.duration_seconds), + cost_measurement="measured_zero_offline", + ) + + @staticmethod + def candidate_resources( + *, + train_set: EvalSet, + validation_set: EvalSet, + prompts: dict[str, str], + eval_config: EvalConfig, + duration_seconds: float, + ) -> ResourceUsage: + """Measure replay evaluation calls, tokens, latency, cost, and time.""" + prompt_text = "\n".join(prompts.values()) + match = re.search( + r"\[variant:\s*([a-zA-Z0-9_-]+)\]", + prompt_text, + re.IGNORECASE, + ) + variant = match.group(1).lower() if match else "baseline" + input_tokens = 0 + output_tokens = 0 + replay_costs: list[float] = [] + replay_cost_complete = True + latencies_ms: list[float] = [] + num_runs = int(eval_config.num_runs) + if num_runs <= 0: + raise ValueError("num_runs must be > 0") + all_cases = train_set.eval_cases + validation_set.eval_cases + for case in all_cases: + state = case.session_input.state if case.session_input else {} + payload = (state.get("variant_traces") or {}).get(variant) or {} + usage = payload.get("usage") or {} + input_tokens += int(usage.get("input_tokens", 0)) + output_tokens += int(usage.get("output_tokens", 0)) + raw_cost = usage.get("cost_usd") + if raw_cost is None: + raw_cost = usage.get("cost") + if raw_cost is None: + replay_cost_complete = False + else: + if isinstance(raw_cost, bool) or not isinstance(raw_cost, (int, float)): + raise ValueError(f"case {case.eval_id!r} has non-numeric replay cost") + case_cost = float(raw_cost) + if not math.isfinite(case_cost) or case_cost < 0: + raise ValueError(f"case {case.eval_id!r} has invalid replay cost") + replay_costs.append(case_cost) + if usage.get("latency_ms") is not None: + latencies_ms.append(float(usage["latency_ms"])) + metric_calls = len(all_cases) * len(eval_config.get_eval_metrics()) * num_runs + judge_metrics = sum(1 for metric in eval_config.get_eval_metrics() if metric.metric_name.startswith("llm_")) + p95_latency_ms = None + if latencies_ms: + latencies_ms.sort() + index = max(0, min(len(latencies_ms) - 1, math.ceil(0.95 * len(latencies_ms)) - 1)) + p95_latency_ms = latencies_ms[index] + measured_cost = math.fsum(replay_costs) * num_runs if replay_cost_complete else None + if measured_cost is None: + cost_measurement = "unavailable" + elif measured_cost == 0.0: + cost_measurement = "measured_zero_offline" + else: + cost_measurement = "measured_from_replay" + return ResourceUsage( + metric_calls=metric_calls, + reflection_calls=0, + judge_calls=len(all_cases) * judge_metrics * num_runs, + prompt_tokens=input_tokens * num_runs, + completion_tokens=output_tokens * num_runs, + total_tokens=(input_tokens + output_tokens) * num_runs, + cost_usd=measured_cost, + duration_seconds=duration_seconds, + p95_latency_ms=p95_latency_ms, + cost_measurement=cost_measurement, + ) + + @staticmethod + def gate( + *, + baseline: BaselineEvaluation, + candidate_train: SplitEvaluation, + candidate_validation: SplitEvaluation, + delta: CandidateDelta, + optimizer_status: str, + resources: ResourceUsage, + config: dict[str, Any], + ) -> GateDecision: + """Apply configured validation, regression, overfit, and budget policy.""" + minimum_gain = float(config.get("min_validation_gain", 0.0)) + gain = delta.validation.pass_rate_delta + newly_failed_ids = set(delta.validation.newly_failed) + candidate_by_id = {case.case_id: case for case in candidate_validation.cases} + new_hard_failures = sorted(case_id for case_id in newly_failed_ids if candidate_by_id[case_id].hard_fail) + epsilon = 1e-12 + baseline_validation_by_id = {case.case_id: case for case in baseline.validation.cases} + key_regressions = sorted( + case.case_id for case in candidate_validation.cases + if case.key_case and baseline_validation_by_id[case.case_id].score - case.score > epsilon) + + def _trend(split_delta: SplitDelta) -> int: + if split_delta.pass_rate_delta > epsilon: + return 1 + if split_delta.pass_rate_delta < -epsilon: + return -1 + if split_delta.average_score_delta > epsilon: + return 1 + if split_delta.average_score_delta < -epsilon: + return -1 + return 0 + + overfitting = _trend(delta.train) > 0 and _trend(delta.validation) < 0 + checks = [ + GateCheck( + name="optimizer_succeeded", + passed=optimizer_status == "SUCCEEDED", + actual=optimizer_status, + expected="SUCCEEDED", + reason="AgentOptimizer must finish successfully.", + ), + GateCheck( + name="minimum_validation_gain", + passed=gain + 1e-12 >= minimum_gain, + actual=gain, + expected=f">= {minimum_gain}", + reason="Candidate validation pass-rate gain must meet the configured floor.", + ), + GateCheck( + name="no_new_hard_failures", + passed=not new_hard_failures, + actual=new_hard_failures, + expected=[], + reason="A baseline pass may not become a hard failure.", + required=bool(config.get("forbid_new_hard_failures", True)), + ), + GateCheck( + name="key_cases_no_regression", + passed=not key_regressions, + actual=key_regressions, + expected=[], + reason="Key validation cases may not lose score.", + required=bool(config.get("key_cases_no_regression", True)), + ), + GateCheck( + name="no_train_validation_overfit", + passed=not overfitting, + actual=overfitting, + expected=False, + reason=("Train improvement paired with validation regression is rejected; " + "pass rate is primary and average score breaks ties."), + required=bool(config.get("reject_overfitting", True)), + ), + ] + ci_floor = config.get("min_validation_gain_ci_lower_bound") + if ci_floor is not None: + lower = delta.validation.paired_pass_rate_ci.lower + checks.append( + GateCheck( + name="validation_gain_ci_lower_bound", + passed=lower + 1e-12 >= float(ci_floor), + actual=lower, + expected=f">= {ci_floor}", + reason=("The paired bootstrap lower bound must meet the " + "configured confidence floor."), + )) + budget = config.get("budget") or {} + for name, actual, key in ( + ("metric_call_budget", resources.metric_calls, "max_metric_calls"), + ("token_budget", resources.total_tokens, "max_total_tokens"), + ("duration_budget", resources.duration_seconds, "max_duration_seconds"), + ): + limit = budget.get(key) + if limit is None: + continue + checks.append( + GateCheck( + name=name, + passed=actual <= float(limit), + actual=actual, + expected=f"<= {limit}", + reason=f"Measured resource usage must stay within {key}.", + )) + max_cost = budget.get("max_cost_usd") + if max_cost is not None: + cost_known = resources.cost_usd is not None and resources.cost_measurement != "unavailable" + if isinstance(max_cost, bool) or not isinstance(max_cost, (int, float)): + raise ValueError("max_cost_usd must be a finite non-negative number") + max_cost_value = float(max_cost) + if not math.isfinite(max_cost_value) or max_cost_value < 0: + raise ValueError("max_cost_usd must be a finite non-negative number") + within_cost_budget = False + if cost_known: + measured_cost = float(resources.cost_usd) + within_cost_budget = measured_cost < max_cost_value or math.isclose( + measured_cost, + max_cost_value, + rel_tol=1e-12, + abs_tol=1e-12, + ) + checks.append( + GateCheck( + name="cost_budget", + passed=within_cost_budget, + actual=resources.cost_usd if cost_known else "unavailable", + expected=f"<= {max_cost}", + reason="Unknown cost fails closed; a measured cost must be within budget.", + )) + accepted = all(check.passed for check in checks if check.required) + return GateDecision( + accepted=accepted, + overfitting_detected=overfitting, + checks=checks, + ) + + @staticmethod + def candidate_id( + prompts: dict[str, str], + known_candidates: dict[str, str], + ) -> str: + """Resolve fixture candidates by exact prompt, otherwise use a stable hash.""" + if len(prompts) == 1: + prompt = next(iter(prompts.values())).strip() + for candidate_id, candidate_prompt in known_candidates.items(): + if candidate_prompt.strip() == prompt: + return candidate_id + return f"candidate-{_sha256_text(json.dumps(prompts, sort_keys=True))[:12]}" + + @staticmethod + def unique_proposals( + proposals: list[tuple[int, dict[str, str]]], + *, + best_prompts: dict[str, str], + ) -> list[tuple[int | None, dict[str, str]]]: + """Deduplicate full prompt maps while retaining optimizer order.""" + unique: list[tuple[int | None, dict[str, str]]] = [] + seen: set[str] = set() + for optimizer_round, prompts in proposals: + fingerprint = _sha256_text(json.dumps(prompts, sort_keys=True, ensure_ascii=False)) + if fingerprint in seen: + continue + seen.add(fingerprint) + unique.append((optimizer_round, dict(prompts))) + if best_prompts: + fingerprint = _sha256_text(json.dumps(best_prompts, sort_keys=True, ensure_ascii=False)) + if fingerprint not in seen: + unique.append((None, dict(best_prompts))) + if not unique: + raise RuntimeError("AgentOptimizer produced no auditable prompt candidate") + return unique + + @staticmethod + def mark_pareto(candidates: list[CandidateEvaluation]) -> None: + """Mark gate-eligible candidates not dominated on quality, tokens, and latency.""" + eligible = [candidate for candidate in candidates if candidate.gate.accepted] + for candidate in candidates: + if not candidate.gate.accepted: + candidate.pareto_optimal = False + continue + dominated = False + for other in eligible: + if other is candidate: + continue + no_worse = (other.validation.pass_rate >= candidate.validation.pass_rate + and other.validation.average_score >= candidate.validation.average_score + and other.audit.resources.total_tokens <= candidate.audit.resources.total_tokens + and (other.audit.resources.p95_latency_ms + or 0.0) <= (candidate.audit.resources.p95_latency_ms or 0.0)) + strictly_better = (other.validation.pass_rate > candidate.validation.pass_rate + or other.validation.average_score > candidate.validation.average_score + or other.audit.resources.total_tokens < candidate.audit.resources.total_tokens + or (other.audit.resources.p95_latency_ms + or 0.0) < (candidate.audit.resources.p95_latency_ms or 0.0)) + if no_worse and strictly_better: + dominated = True + break + candidate.pareto_optimal = not dominated + + @staticmethod + def combined_rejection_gate(candidates: list[CandidateEvaluation]) -> GateDecision: + """Return representative rejection evidence when no candidate passes.""" + if candidates: + return candidates[-1].gate + return GateDecision( + accepted=False, + checks=[ + GateCheck( + name="candidate_available", + passed=False, + actual=0, + expected=">= 1", + reason="No independently evaluable candidate was produced.", + ) + ], + ) + + @staticmethod + def failure_summary(scoped_cases: dict[str, list[CaseEvaluation]], ) -> FailureAttributionSummary: + """Summarize evidence coverage and categories across every evaluated scope.""" + failures = [(scope, case) for scope, cases in scoped_cases.items() for case in cases if not case.passed] + by_case = {f"{scope}/{case.case_id}": case.failure_reasons for scope, case in failures if case.failure_reasons} + counts = Counter(reason.category for reasons in by_case.values() for reason in reasons) + explained = len(by_case) + total = len(failures) + return FailureAttributionSummary( + explained_failed_cases=explained, + total_failed_cases=total, + coverage_rate=explained / total if total else 1.0, + category_counts=dict(sorted(counts.items())), + by_case=by_case, + ) diff --git a/examples/optimization/eval_optimize_loop/loop/models.py b/examples/optimization/eval_optimize_loop/loop/models.py new file mode 100644 index 00000000..ee3d1951 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/loop/models.py @@ -0,0 +1,289 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Configuration and report models for the closed-loop example.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from typing import Literal +from typing import Optional + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Field + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class CandidateSource(_StrictModel): + """A deterministic prompt proposal used by the offline reflection model.""" + + candidate_id: str + path: Path + + +class PipelineSpec(_StrictModel): + """All paths and run controls needed by :class:`EvalOptimizePipeline`.""" + + manifest_path: Path + optimizer_config: Path + regression_metrics_config: Path + gate_config: Path + train_dataset: Path + validation_dataset: Path + target_prompts: dict[str, Path] + candidate_sources: list[CandidateSource] + output_dir: Path + seed: int = 91 + bootstrap_samples: int = Field(default=2000, ge=100) + confidence_level: float = Field(default=0.95, gt=0.0, lt=1.0) + apply_if_accepted: bool = False + + @classmethod + def from_file( + cls, + path: str | Path, + *, + output_dir: str | Path | None = None, + ) -> "PipelineSpec": + """Load a manifest and resolve every relative path beside it.""" + manifest_path = Path(path).expanduser().resolve() + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + root = manifest_path.parent + + def _resolve(value: str | Path) -> Path: + candidate = Path(value).expanduser() + if not candidate.is_absolute(): + candidate = root / candidate + return candidate.resolve() + + if output_dir is None: + configured_output = _resolve(payload.get("output_dir", "runs/latest")) + else: + configured_output = Path(output_dir).expanduser().resolve() + payload.pop("output_dir", None) + candidates = [ + CandidateSource( + candidate_id=item["candidate_id"], + path=_resolve(item["path"]), + ) for item in payload.pop("candidate_sources") + ] + target_prompts = {name: _resolve(prompt_path) for name, prompt_path in payload.pop("target_prompts").items()} + return cls( + manifest_path=manifest_path, + optimizer_config=_resolve(payload.pop("optimizer_config")), + regression_metrics_config=_resolve(payload.pop("regression_metrics_config")), + gate_config=_resolve(payload.pop("gate_config")), + train_dataset=_resolve(payload.pop("train_dataset")), + validation_dataset=_resolve(payload.pop("validation_dataset")), + target_prompts=target_prompts, + candidate_sources=candidates, + output_dir=configured_output, + **payload, + ) + + +class MetricOutcome(_StrictModel): + metric_name: str + score: Optional[float] = None + threshold: float + passed: bool + reason: str = "" + + +class FailureReason(_StrictModel): + category: str + explanation: str + evidence: dict[str, Any] = Field(default_factory=dict) + + +class TrajectoryStep(_StrictModel): + kind: str + name: str + payload: dict[str, Any] = Field(default_factory=dict) + + +class CaseEvaluation(_StrictModel): + case_id: str + passed: bool + score: float + key_case: bool = False + hard_fail: bool = False + metrics: list[MetricOutcome] = Field(default_factory=list) + primary_failure: Optional[str] = None + failure_reasons: list[FailureReason] = Field(default_factory=list) + actual_response: str = "" + expected_response: str = "" + key_trajectory: list[TrajectoryStep] = Field(default_factory=list) + + +class SplitEvaluation(_StrictModel): + split: Literal["train", "validation"] + pass_rate: float + average_score: float + cases: list[CaseEvaluation] + + +class BaselineEvaluation(_StrictModel): + train: SplitEvaluation + validation: SplitEvaluation + + +DeltaStatus = Literal[ + "newly_passed", + "newly_failed", + "score_improved", + "score_regressed", + "unchanged", +] + + +class CaseDelta(_StrictModel): + case_id: str + status: DeltaStatus + baseline_passed: bool + candidate_passed: bool + baseline_score: float + candidate_score: float + score_delta: float + + +class PairedConfidenceInterval(_StrictModel): + point_estimate: float + lower: float + upper: float + confidence_level: float + bootstrap_samples: int + seed: int + + +class SplitDelta(_StrictModel): + split: Literal["train", "validation"] + pass_rate_delta: float + average_score_delta: float + paired_pass_rate_ci: PairedConfidenceInterval + newly_passed: list[str] = Field(default_factory=list) + newly_failed: list[str] = Field(default_factory=list) + score_improved: list[str] = Field(default_factory=list) + score_regressed: list[str] = Field(default_factory=list) + unchanged: list[str] = Field(default_factory=list) + cases: list[CaseDelta] = Field(default_factory=list) + + +class CandidateDelta(_StrictModel): + train: SplitDelta + validation: SplitDelta + + +class GateCheck(_StrictModel): + name: str + passed: bool + actual: Any + expected: Any + reason: str + required: bool = True + + +class GateDecision(_StrictModel): + accepted: bool + overfitting_detected: bool = False + checks: list[GateCheck] = Field(default_factory=list) + + +class ResourceUsage(_StrictModel): + metric_calls: int = 0 + reflection_calls: int = 0 + judge_calls: Optional[int] = None + prompt_tokens: int = 0 + completion_tokens: int = 0 + total_tokens: int = 0 + cost_usd: Optional[float] = None + duration_seconds: float = 0.0 + p95_latency_ms: Optional[float] = None + cost_measurement: str = "unavailable" + + +class CandidateAudit(_StrictModel): + prompt_sha256: str + source: str + optimizer_round: Optional[int] = None + seed: int + resources: ResourceUsage + + +class CandidateEvaluation(_StrictModel): + candidate_id: str + prompts: dict[str, str] + train: SplitEvaluation + validation: SplitEvaluation + delta: CandidateDelta + gate: GateDecision + audit: CandidateAudit + pareto_optimal: bool = False + + +class OptimizerAudit(_StrictModel): + algorithm: str + status: str + stop_reason: Optional[str] = None + used_agent_optimizer: bool + baseline_pass_rate: float + best_pass_rate: float + rounds: int + resources: ResourceUsage + artifact_dir: str + + +class DataQualityAudit(_StrictModel): + passed: bool + train_cases: int + validation_cases: int + duplicate_ids: list[str] = Field(default_factory=list) + cross_split_duplicates: list[str] = Field(default_factory=list) + near_cross_split_duplicates: list[str] = Field(default_factory=list) + prompt_leakage_matches: list[str] = Field(default_factory=list) + + +class RunAudit(_StrictModel): + run_id: str + started_at: str + finished_at: str + duration_seconds: float + seed: int + config_sha256: str + train_sha256: str + validation_sha256: str + baseline_prompt_sha256: dict[str, str] + command: str + + +class FailureAttributionSummary(_StrictModel): + explained_failed_cases: int + total_failed_cases: int + coverage_rate: float + category_counts: dict[str, int] = Field(default_factory=dict) + by_case: dict[str, list[FailureReason]] = Field(default_factory=dict) + + +class OptimizationReport(_StrictModel): + schema_version: str = "1.0" + status: Literal["accepted", "rejected", "failed"] + baseline: BaselineEvaluation + candidates: list[CandidateEvaluation] + selected_candidate_id: Optional[str] = None + candidate: Optional[CandidateEvaluation] = None + delta: Optional[CandidateDelta] = None + gate: Optional[GateDecision] = None + failure_attribution: FailureAttributionSummary + optimizer: OptimizerAudit + data_quality: DataQualityAudit + audit: RunAudit diff --git a/examples/optimization/eval_optimize_loop/loop/offline.py b/examples/optimization/eval_optimize_loop/loop/offline.py new file mode 100644 index 00000000..004e7069 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/loop/offline.py @@ -0,0 +1,294 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Patch-free offline adapters for Agent, judge, reflection, and trace replay.""" + +from __future__ import annotations + +import json +import re +import threading +import uuid +from collections.abc import AsyncGenerator +from pathlib import Path +from typing import Any + +from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.evaluation import EvalCase +from trpc_agent_sdk.evaluation import EvalSet +from trpc_agent_sdk.models import LLMModel +from trpc_agent_sdk.models import LlmRequest +from trpc_agent_sdk.models import LlmResponse +from trpc_agent_sdk.models import ModelRegistry +from trpc_agent_sdk.runners import Runner +from trpc_agent_sdk.sessions import InMemorySessionService +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import GenerateContentResponseUsageMetadata +from trpc_agent_sdk.types import Part + +_VARIANT_RE = re.compile(r"\[variant:\s*([a-zA-Z0-9_-]+)\]", re.IGNORECASE) + + +def _content_text(content: Any) -> str: + if content is None: + return "" + if isinstance(content, str): + return content + parts = getattr(content, "parts", None) or [] + return "\n".join(str(getattr(part, "text", "") or "") for part in parts).strip() + + +def _invocation_response_text(payload: dict[str, Any]) -> str: + response = payload.get("final_response") or {} + return "\n".join(str(part.get("text", "")) for part in response.get("parts", [])).strip() + + +def _request_text(request: LlmRequest) -> str: + return "\n".join(_content_text(content) for content in (request.contents or [])).strip() + + +def _system_text(request: LlmRequest) -> str: + config = request.config + if config is None: + return "" + return _content_text(getattr(config, "system_instruction", None)) + + +def _tag(text: str, name: str) -> str: + match = re.search( + rf"<{re.escape(name)}>\s*(.*?)\s*", + text, + re.DOTALL | re.IGNORECASE, + ) + return match.group(1).strip() if match else "" + + +class OfflineModel(LLMModel): + """One deterministic provider with agent, judge, and reflector model names.""" + + _catalog: dict[str, EvalCase] = {} + _candidate_prompts: list[str] = [] + _configuration_lock = threading.Lock() + + @classmethod + def supported_models(cls) -> list[str]: + return [r"offline/.*"] + + @classmethod + def configure( + cls, + *, + eval_sets: list[EvalSet], + candidate_prompts: list[str], + ) -> None: + """Replace immutable replay inputs before a pipeline run.""" + catalog: dict[str, EvalCase] = {} + for eval_set in eval_sets: + for case in eval_set.eval_cases: + conversation = case.conversation or [] + if not conversation: + continue + query = _content_text(conversation[0].user_content) + catalog[query] = case + with cls._configuration_lock: + cls._catalog = catalog + cls._candidate_prompts = list(candidate_prompts) + + def __init__(self, model_name: str, **kwargs: Any) -> None: + super().__init__(model_name=model_name, **kwargs) + self._reflection_index = 0 + self._reflection_lock = threading.Lock() + + def validate_request(self, request: LlmRequest) -> None: + if not request.contents: + raise ValueError("offline model requires at least one content item") + + async def _generate_async_impl( + self, + request: LlmRequest, + stream: bool = False, + ctx: Any = None, + ) -> AsyncGenerator[LlmResponse, None]: + del stream, ctx + if self.name == "offline/agent": + text = self._agent_response(request) + elif self.name == "offline/judge": + text = self._judge_response(request) + elif self.name == "offline/reflector": + text = self._reflection_response() + else: + raise ValueError(f"unsupported offline model: {self.name}") + + prompt_text = "\n\n".join(item for item in (_system_text(request), _request_text(request)) if item) + prompt_tokens = max(1, len(prompt_text) // 4) + completion_tokens = max(1, len(text) // 4) + usage = GenerateContentResponseUsageMetadata( + prompt_token_count=prompt_tokens, + candidates_token_count=completion_tokens, + total_token_count=prompt_tokens + completion_tokens, + ) + yield LlmResponse( + content=Content(role="model", parts=[Part.from_text(text=text)]), + usage_metadata=usage, + partial=False, + ) + + def _agent_response(self, request: LlmRequest) -> str: + query = _request_text(request) + case = self._catalog.get(query) + if case is None: + raise ValueError(f"offline replay has no case for query: {query!r}") + variant_match = _VARIANT_RE.search(_system_text(request)) + variant = variant_match.group(1).lower() if variant_match else "baseline" + state = case.session_input.state if case.session_input else {} + traces = state.get("variant_traces") or {} + payload = traces.get(variant) or traces.get("baseline") + if not isinstance(payload, dict): + raise ValueError(f"case {case.eval_id!r} has no trace for variant {variant!r}") + return _invocation_response_text(payload) + + def _reflection_response(self) -> str: + with self._reflection_lock: + if not self._candidate_prompts: + raise ValueError("offline reflector has no configured candidate prompts") + index = min(self._reflection_index, len(self._candidate_prompts) - 1) + self._reflection_index += 1 + prompt = self._candidate_prompts[index] + return f"```\n{prompt.rstrip()}\n```" + + def _judge_response(self, request: LlmRequest) -> str: + prompt = _request_text(request) + query = _tag(_tag(prompt, "user_prompt"), "main_prompt") + case = self._catalog.get(query) + rubrics_text = _tag(prompt, "rubric") + rubric_lines = [line.strip() for line in rubrics_text.splitlines() if ":" in line] + signals = self._resolve_signals(case, prompt) + items: list[dict[str, str]] = [] + for line in rubric_lines: + rubric_id, rubric = line.split(":", 1) + normalized = rubric_id.strip().lower() + if "format" in normalized: + passed = signals.get("format_pass", False) + elif "knowledge" in normalized or "recall" in normalized: + passed = signals.get("knowledge_recall_pass", False) + else: + passed = signals.get("llm_rubric_pass", False) + items.append({ + "id": + rubric_id.strip(), + "rubric": + rubric.strip(), + "evidence": + "offline deterministic replay signal", + "reason": ("replay evidence satisfies rubric" if passed else "replay evidence does not satisfy rubric"), + "verdict": + "yes" if passed else "no", + }) + if not items: + items.append({ + "id": "offline", + "rubric": "offline fallback", + "evidence": "no rubric line parsed", + "reason": "offline fallback rejects malformed judge prompt", + "verdict": "no", + }) + return json.dumps({"items": items}, ensure_ascii=False) + + @classmethod + def _resolve_signals(cls, case: EvalCase | None, judge_prompt: str) -> dict[str, bool]: + if case is None or case.session_input is None: + return {} + state = case.session_input.state + traces = state.get("variant_traces") or {} + final_answer = _tag(judge_prompt, "final_answer") + retrieved = _tag(judge_prompt, "retrieved_knowledge") + for payload in traces.values(): + if not isinstance(payload, dict): + continue + response_matches = final_answer and _invocation_response_text(payload) == final_answer + knowledge_matches = False + intermediate = payload.get("intermediate_data") or {} + for tool_response in intermediate.get("tool_responses") or []: + encoded = json.dumps(tool_response.get("response"), ensure_ascii=False) + if encoded and encoded in retrieved: + knowledge_matches = True + break + if response_matches or knowledge_matches: + raw = payload.get("signals") or {} + return { + "format_pass": bool(raw.get("format_pass", True)), + "knowledge_recall_pass": bool(raw.get("knowledge_recall_pass", True)), + "llm_rubric_pass": bool(raw.get("llm_rubric_pass", True)), + } + expected_tools = [] + for invocation in case.conversation or []: + intermediate = invocation.intermediate_data + if intermediate is not None: + expected_tools.extend(getattr(tool, "name", "") for tool in getattr(intermediate, "tool_uses", [])) + if _tag(judge_prompt, "retrieved_knowledge") and "search_policy" not in expected_tools: + return { + "format_pass": True, + "knowledge_recall_pass": True, + "llm_rubric_pass": True, + } + return {} + + +def configure_offline_models( + *, + eval_sets: list[EvalSet], + candidate_prompts: list[str], +) -> None: + """Register the provider and configure a fresh deterministic replay.""" + ModelRegistry.register(OfflineModel) + ModelRegistry.resolve.cache_clear() + OfflineModel.configure( + eval_sets=eval_sets, + candidate_prompts=candidate_prompts, + ) + + +def create_offline_call_agent(prompt_paths: dict[str, Path]): + """Build the optimizer's async ``call_agent`` using a real :class:`LlmAgent`.""" + + async def call_agent(query: str) -> str: + instructions = [] + for name, path in prompt_paths.items(): + instructions.append(f"## {name}\n{path.read_text(encoding='utf-8')}") + root_agent = LlmAgent( + name="offline_support_agent", + model=OfflineModel(model_name="offline/agent"), + instruction="\n\n".join(instructions), + add_name_to_instruction=False, + ) + session_service = InMemorySessionService() + runner = Runner( + app_name="eval_optimize_loop", + agent=root_agent, + session_service=session_service, + ) + session_id = str(uuid.uuid4()) + await session_service.create_session( + app_name="eval_optimize_loop", + user_id="offline", + session_id=session_id, + state={}, + ) + final_text = "" + async for event in runner.run_async( + user_id="offline", + session_id=session_id, + new_message=Content(role="user", parts=[Part.from_text(text=query)]), + ): + if not event.is_final_response() or not event.content: + continue + for part in event.content.parts or []: + if not part.thought and part.text: + final_text += part.text + return final_text.strip() + + return call_agent diff --git a/examples/optimization/eval_optimize_loop/loop/pipeline.py b/examples/optimization/eval_optimize_loop/loop/pipeline.py new file mode 100644 index 00000000..e609ad92 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/loop/pipeline.py @@ -0,0 +1,284 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""End-to-end orchestration for the evaluation and optimization closed loop.""" + +from __future__ import annotations + +import hashlib +import json +import time +import uuid +from datetime import datetime +from datetime import timezone +from pathlib import Path +from typing import Any + +from trpc_agent_sdk.evaluation import AgentOptimizer +from trpc_agent_sdk.evaluation import EvalConfig +from trpc_agent_sdk.evaluation import EvalSet +from trpc_agent_sdk.evaluation import TargetPrompt + +from .analysis import RegressionAnalyzer +from .models import BaselineEvaluation +from .models import CandidateAudit +from .models import CandidateDelta +from .models import CandidateEvaluation +from .models import OptimizationReport +from .models import OptimizerAudit +from .models import PipelineSpec +from .models import RunAudit +from .offline import configure_offline_models +from .offline import create_offline_call_agent +from .reporting import write_reports +from .trace import TraceEvaluator + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _sha256_file(path: Path) -> str: + return _sha256_bytes(path.read_bytes()) + + +def _sha256_text(text: str) -> str: + return _sha256_bytes(text.encode("utf-8")) + + +class _ProposalCapture: + """Observe every reflection proposal, including ones GEPA later rejects.""" + + def __init__(self) -> None: + self._parents: dict[int, dict[str, str]] = {} + self.proposals: list[tuple[int, dict[str, str]]] = [] + + def on_proposal_start(self, event: dict[str, Any]) -> None: + self._parents[int(event["iteration"])] = dict(event["parent_candidate"]) + + def on_proposal_end(self, event: dict[str, Any]) -> None: + iteration = int(event["iteration"]) + candidate = dict(self._parents.get(iteration, {})) + candidate.update(dict(event.get("new_instructions") or {})) + if candidate: + self.proposals.append((iteration, candidate)) + + +class EvalOptimizePipeline: + """A deep module whose only operation executes the complete closed loop.""" + + def __init__(self, spec: PipelineSpec) -> None: + self._spec = spec + + async def run(self) -> OptimizationReport: + """Run baseline, optimization, independent regression, gate, and reports.""" + started_clock = time.perf_counter() + started_at = datetime.now(timezone.utc) + spec = self._spec + spec.output_dir.mkdir(parents=True, exist_ok=True) + + analyzer = RegressionAnalyzer( + seed=spec.seed, + bootstrap_samples=spec.bootstrap_samples, + confidence_level=spec.confidence_level, + ) + trace_evaluator = TraceEvaluator(spec.output_dir) + train_set = EvalSet.model_validate_json(spec.train_dataset.read_text(encoding="utf-8")) + validation_set = EvalSet.model_validate_json(spec.validation_dataset.read_text(encoding="utf-8")) + known_candidates = { + source.candidate_id: source.path.read_text(encoding="utf-8") + for source in spec.candidate_sources + } + prompt_text = "\n".join( + path.read_text(encoding="utf-8") + for path in (list(spec.target_prompts.values()) + [source.path for source in spec.candidate_sources])) + data_quality = analyzer.validate_data_quality( + train_set, + validation_set, + train_path=spec.train_dataset, + validation_path=spec.validation_dataset, + prompt_text=prompt_text, + ) + regression_config = EvalConfig.model_validate_json(spec.regression_metrics_config.read_text(encoding="utf-8")) + gate_config = json.loads(spec.gate_config.read_text(encoding="utf-8")) + + configure_offline_models( + eval_sets=[train_set, validation_set], + candidate_prompts=list(known_candidates.values()), + ) + target_prompt = TargetPrompt() + for name, path in spec.target_prompts.items(): + target_prompt.add_path(name, str(path)) + baseline_prompts = await target_prompt.read_all() + call_agent = create_offline_call_agent(spec.target_prompts) + + baseline = BaselineEvaluation( + train=await trace_evaluator.evaluate( + train_set, + split="train", + eval_config=regression_config, + prompts=baseline_prompts, + trace_label="baseline", + ), + validation=await trace_evaluator.evaluate( + validation_set, + split="validation", + eval_config=regression_config, + prompts=baseline_prompts, + trace_label="baseline", + ), + ) + + optimizer_dir = spec.output_dir / "optimizer" + proposal_capture = _ProposalCapture() + optimize_result = await AgentOptimizer.optimize( + config_path=str(spec.optimizer_config), + call_agent=call_agent, + target_prompt=target_prompt, + train_dataset_path=str(spec.train_dataset), + validation_dataset_path=str(spec.validation_dataset), + output_dir=str(optimizer_dir), + update_source=False, + verbose=0, + extra_gepa_callbacks=[proposal_capture], + ) + proposal_records = analyzer.unique_proposals( + proposal_capture.proposals, + best_prompts=dict(optimize_result.best_prompts), + ) + optimizer_resources = analyzer.optimizer_resources(optimize_result) + + candidates: list[CandidateEvaluation] = [] + try: + for optimizer_round, prompts in proposal_records: + candidate_started = time.perf_counter() + candidate_id = analyzer.candidate_id(prompts, known_candidates) + await target_prompt.write_all(prompts) + candidate_train = await trace_evaluator.evaluate( + train_set, + split="train", + eval_config=regression_config, + prompts=prompts, + trace_label=candidate_id, + ) + candidate_validation = await trace_evaluator.evaluate( + validation_set, + split="validation", + eval_config=regression_config, + prompts=prompts, + trace_label=candidate_id, + ) + delta = CandidateDelta( + train=analyzer.diff(baseline.train, candidate_train), + validation=analyzer.diff(baseline.validation, candidate_validation), + ) + candidate_resources = analyzer.candidate_resources( + train_set=train_set, + validation_set=validation_set, + prompts=prompts, + eval_config=regression_config, + duration_seconds=time.perf_counter() - candidate_started, + ) + gate = analyzer.gate( + baseline=baseline, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + delta=delta, + optimizer_status=optimize_result.status, + resources=candidate_resources, + config=gate_config, + ) + candidates.append( + CandidateEvaluation( + candidate_id=candidate_id, + prompts=prompts, + train=candidate_train, + validation=candidate_validation, + delta=delta, + gate=gate, + audit=CandidateAudit( + prompt_sha256=_sha256_text(json.dumps(prompts, sort_keys=True, ensure_ascii=False)), + source="GEPA on_proposal_end", + optimizer_round=optimizer_round, + seed=spec.seed, + resources=candidate_resources, + ), + )) + finally: + await target_prompt.write_all(baseline_prompts) + + analyzer.mark_pareto(candidates) + accepted_candidates = [candidate for candidate in candidates if candidate.gate.accepted] + selected = (max( + accepted_candidates, + key=lambda item: ( + item.validation.pass_rate, + item.validation.average_score, + -item.audit.resources.total_tokens, + item.candidate_id, + ), + ) if accepted_candidates else None) + if selected is not None and spec.apply_if_accepted: + await target_prompt.write_all(selected.prompts) + + scoped_cases = { + "baseline/train": baseline.train.cases, + "baseline/validation": baseline.validation.cases, + } + for candidate_item in candidates: + scoped_cases[f"candidate/{candidate_item.candidate_id}/train"] = candidate_item.train.cases + scoped_cases[f"candidate/{candidate_item.candidate_id}/validation"] = candidate_item.validation.cases + failure_summary = analyzer.failure_summary(scoped_cases) + + finished_at = datetime.now(timezone.utc) + duration = time.perf_counter() - started_clock + status = "accepted" if selected is not None else "rejected" + report = OptimizationReport( + status=status, + baseline=baseline, + candidates=candidates, + selected_candidate_id=selected.candidate_id if selected is not None else None, + candidate=selected, + delta=selected.delta if selected is not None else None, + gate=selected.gate if selected is not None else analyzer.combined_rejection_gate(candidates), + failure_attribution=failure_summary, + optimizer=OptimizerAudit( + algorithm=optimize_result.algorithm, + status=optimize_result.status, + stop_reason=optimize_result.stop_reason, + used_agent_optimizer=True, + baseline_pass_rate=optimize_result.baseline_pass_rate, + best_pass_rate=optimize_result.best_pass_rate, + rounds=optimize_result.total_rounds, + resources=optimizer_resources, + artifact_dir="optimizer", + ), + data_quality=data_quality, + audit=RunAudit( + run_id=f"offline-{uuid.uuid4().hex[:12]}", + started_at=started_at.isoformat(), + finished_at=finished_at.isoformat(), + duration_seconds=duration, + seed=spec.seed, + config_sha256=_sha256_text("".join( + path.read_text(encoding="utf-8") for path in ( + spec.manifest_path, + spec.optimizer_config, + spec.regression_metrics_config, + spec.gate_config, + ))), + train_sha256=_sha256_file(spec.train_dataset), + validation_sha256=_sha256_file(spec.validation_dataset), + baseline_prompt_sha256={ + name: _sha256_text(text) + for name, text in baseline_prompts.items() + }, + command="python examples/optimization/eval_optimize_loop/run_pipeline.py", + ), + ) + write_reports(report, spec.output_dir) + return report diff --git a/examples/optimization/eval_optimize_loop/loop/reporting.py b/examples/optimization/eval_optimize_loop/loop/reporting.py new file mode 100644 index 00000000..c21cd0f8 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/loop/reporting.py @@ -0,0 +1,152 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Atomic JSON and Markdown report persistence.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from .models import OptimizationReport + + +def _atomic_write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp") + temporary.write_text(text, encoding="utf-8") + os.replace(temporary, path) + + +def write_reports(report: OptimizationReport, output_dir: Path) -> None: + """Persist both report formats without exposing partial files.""" + json_text = report.model_dump_json(indent=2) + "\n" + _atomic_write(output_dir / "optimization_report.json", json_text) + _atomic_write(output_dir / "optimization_report.md", render_markdown(report)) + + +def render_markdown(report: OptimizationReport) -> str: + """Render the decision-first human report.""" + selected = report.candidate + lines = [ + "# Evaluation + Optimization Regression Report", + "", + f"- **Decision:** `{report.status}`", + f"- **Selected candidate:** `{report.selected_candidate_id or 'none'}`", + f"- **Optimizer:** `{report.optimizer.algorithm}` / `{report.optimizer.status}`", + f"- **Seed:** `{report.audit.seed}`", + "", + "## Baseline", + "", + "| Split | Pass rate | Average score | Cases |", + "| --- | ---: | ---: | ---: |", + (f"| Train | {report.baseline.train.pass_rate:.3f} | " + f"{report.baseline.train.average_score:.3f} | {len(report.baseline.train.cases)} |"), + (f"| Validation | {report.baseline.validation.pass_rate:.3f} | " + f"{report.baseline.validation.average_score:.3f} | " + f"{len(report.baseline.validation.cases)} |"), + "", + "## Candidate matrix", + "", + ("| Candidate | Train pass | Validation pass | Validation delta | " + "Paired CI | Gate | Overfit | Pareto |"), + "| --- | ---: | ---: | ---: | --- | --- | --- | --- |", + ] + for item in report.candidates: + ci = item.delta.validation.paired_pass_rate_ci + lines.append(f"| `{item.candidate_id}` | {item.train.pass_rate:.3f} | " + f"{item.validation.pass_rate:.3f} | {item.delta.validation.pass_rate_delta:+.3f} | " + f"[{ci.lower:+.3f}, {ci.upper:+.3f}] | " + f"{'PASS' if item.gate.accepted else 'FAIL'} | " + f"{item.gate.overfitting_detected} | {item.pareto_optimal} |") + lines.extend([ + "", + "## Candidate", + "", + ]) + if selected is None: + lines.append("No candidate passed the configured gate.") + else: + lines.extend([ + f"Candidate `{selected.candidate_id}` was independently re-evaluated.", + "", + "| Split | Baseline | Candidate | Delta |", + "| --- | ---: | ---: | ---: |", + (f"| Train pass rate | {report.baseline.train.pass_rate:.3f} | " + f"{selected.train.pass_rate:.3f} | {selected.delta.train.pass_rate_delta:+.3f} |"), + (f"| Validation pass rate | {report.baseline.validation.pass_rate:.3f} | " + f"{selected.validation.pass_rate:.3f} | " + f"{selected.delta.validation.pass_rate_delta:+.3f} |"), + "", + ("Paired bootstrap interval: " + f"`[{selected.delta.validation.paired_pass_rate_ci.lower:+.3f}, " + f"{selected.delta.validation.paired_pass_rate_ci.upper:+.3f}]` " + f"at {selected.delta.validation.paired_pass_rate_ci.confidence_level:.0%} " + "confidence."), + "", + "### Per-case delta", + "", + "| Case | Split | Status | Score delta |", + "| --- | --- | --- | ---: |", + ]) + for split_delta in (selected.delta.train, selected.delta.validation): + for case in split_delta.cases: + lines.append(f"| `{case.case_id}` | {split_delta.split} | {case.status} | " + f"{case.score_delta:+.3f} |") + + lines.extend([ + "", + "## Gate", + "", + "| Candidate | Check | Required | Result | Actual | Expected |", + "| --- | --- | --- | --- | --- | --- |", + ]) + for item in report.candidates: + for check in item.gate.checks: + lines.append(f"| `{item.candidate_id}` | `{check.name}` | {check.required} | " + f"{'PASS' if check.passed else 'FAIL'} | `{check.actual}` | " + f"`{check.expected}` |") + + lines.extend([ + "", + "## Failure attribution", + "", + (f"Explained {report.failure_attribution.explained_failed_cases}/" + f"{report.failure_attribution.total_failed_cases} failed cases " + f"({report.failure_attribution.coverage_rate:.1%})."), + "", + "| Case | Category | Explanation |", + "| --- | --- | --- |", + ]) + for case_id, reasons in report.failure_attribution.by_case.items(): + for reason in reasons: + lines.append(f"| `{case_id}` | `{reason.category}` | {reason.explanation} |") + + lines.extend([ + "", + "## Audit", + "", + f"- Run id: `{report.audit.run_id}`", + f"- Duration: `{report.audit.duration_seconds:.3f}s`", + f"- Metric calls: `{report.optimizer.resources.metric_calls}`", + f"- Tokens: `{report.optimizer.resources.total_tokens}`", + f"- Cost measurement: `{report.optimizer.resources.cost_measurement}`", + f"- Optimizer artifacts: `{report.optimizer.artifact_dir}`", + "", + "### Candidate evaluation resources", + "", + "| Candidate | Metric calls | Judge calls | Tokens | P95 latency | Duration | Cost | Cost measurement |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + ]) + for item in report.candidates: + usage = item.audit.resources + cost = "unavailable" if usage.cost_usd is None else f"${usage.cost_usd:.4f}" + lines.append(f"| `{item.candidate_id}` | {usage.metric_calls} | {usage.judge_calls} | " + f"{usage.total_tokens} | {usage.p95_latency_ms or 0.0:.1f} ms | " + f"{usage.duration_seconds:.3f} s | {cost} | `{usage.cost_measurement}` |") + lines.append("") + return "\n".join(lines) diff --git a/examples/optimization/eval_optimize_loop/loop/trace.py b/examples/optimization/eval_optimize_loop/loop/trace.py new file mode 100644 index 00000000..d934dc31 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/loop/trace.py @@ -0,0 +1,543 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Trace replay, SDK evaluation, result normalization, and failure attribution.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +from trpc_agent_sdk.evaluation import AgentEvaluator +from trpc_agent_sdk.evaluation import EvalCase +from trpc_agent_sdk.evaluation import EvalCaseResult +from trpc_agent_sdk.evaluation import EvalConfig +from trpc_agent_sdk.evaluation import EvalSet +from trpc_agent_sdk.evaluation import EvalStatus +from trpc_agent_sdk.evaluation import IntermediateData +from trpc_agent_sdk.evaluation import Invocation +from trpc_agent_sdk.evaluation import get_all_tool_calls +from trpc_agent_sdk.evaluation import get_all_tool_responses + +from .models import CaseEvaluation +from .models import FailureReason +from .models import MetricOutcome +from .models import SplitEvaluation + + +def _text(content: Any) -> str: + if content is None: + return "" + return "\n".join(str(getattr(part, "text", "") or "") for part in (content.parts or [])).strip() + + +def _payload_response_text(payload: dict[str, Any]) -> str: + response = payload.get("final_response") or {} + return "\n".join(str(part.get("text", "") or "") for part in response.get("parts", [])).strip() + + +class TraceEvaluator: + """Replay deterministic traces through AgentEvaluator and explain failures.""" + + def __init__(self, output_dir: Path) -> None: + self._output_dir = output_dir + + async def evaluate( + self, + eval_set: EvalSet, + *, + split: str, + eval_config: EvalConfig, + prompts: dict[str, str], + trace_label: str, + ) -> SplitEvaluation: + """Materialize, persist, evaluate, and normalize one replay split.""" + trace_set = self._materialize_trace(eval_set, prompts=prompts, trace_label=trace_label) + trace_dir = self._output_dir / "traces" / trace_label + trace_dir.mkdir(parents=True, exist_ok=True) + trace_path = trace_dir / f"{split}.trace.evalset.json" + trace_path.write_text( + trace_set.model_dump_json(indent=2, by_alias=True) + "\n", + encoding="utf-8", + ) + _, _, _, raw_results = await AgentEvaluator.evaluate_eval_set( + trace_set, + eval_config=eval_config, + num_runs=eval_config.num_runs, + print_detailed_results=False, + case_parallelism=1, + case_eval_parallelism=1, + ) + cases = [self._normalize_case(case, raw_results.get(case.eval_id, [])) for case in trace_set.eval_cases] + pass_rate = sum(1 for case in cases if case.passed) / len(cases) + average_score = sum(case.score for case in cases) / len(cases) + return SplitEvaluation( + split=split, + pass_rate=pass_rate, + average_score=average_score, + cases=cases, + ) + + @staticmethod + def _materialize_trace( + eval_set: EvalSet, + *, + prompts: dict[str, str], + trace_label: str, + ) -> EvalSet: + prompt_text = "\n".join(prompts.values()) + match = re.search( + r"\[variant:\s*([a-zA-Z0-9_-]+)\]", + prompt_text, + re.IGNORECASE, + ) + variant = match.group(1).lower() if match else "baseline" + trace_cases: list[EvalCase] = [] + for case in eval_set.eval_cases: + conversation = case.conversation or [] + if not conversation: + raise ValueError(f"case {case.eval_id!r} has no expected conversation") + state = case.session_input.state if case.session_input else {} + payload = (state.get("variant_traces") or {}).get(variant) + if not isinstance(payload, dict): + raise ValueError(f"case {case.eval_id!r} has no replay trace for variant {variant!r}") + expected_invocation = conversation[0] + actual = Invocation( + invocation_id=f"{trace_label}-{case.eval_id}", + user_content=expected_invocation.user_content, + final_response=payload.get("final_response"), + intermediate_data=IntermediateData.model_validate(payload.get("intermediate_data") or {}), + ) + trace_cases.append( + EvalCase( + eval_id=case.eval_id, + eval_mode="trace", + conversation=case.conversation, + actual_conversation=[actual], + session_input=case.session_input, + )) + return EvalSet( + eval_set_id=f"{eval_set.eval_set_id}_{trace_label}", + name=eval_set.name, + description=eval_set.description, + eval_cases=trace_cases, + ) + + def _normalize_case(self, case: EvalCase, runs: list[EvalCaseResult]) -> CaseEvaluation: + metric_values: dict[str, list[float]] = {} + metric_templates: dict[str, Any] = {} + for run in runs: + for metric in run.overall_eval_metric_results or []: + if metric.score is not None: + metric_values.setdefault(metric.metric_name, []).append(float(metric.score)) + metric_templates[metric.metric_name] = metric + + metrics: list[MetricOutcome] = [] + for name, metric in metric_templates.items(): + scores = metric_values.get(name, []) + score = sum(scores) / len(scores) if scores else None + passed = score is not None and score >= float(metric.threshold) + reason = metric.details.reason if metric.details and metric.details.reason else "" + metrics.append( + MetricOutcome( + metric_name=name, + score=score, + threshold=float(metric.threshold), + passed=passed, + reason=reason, + )) + + run_passed = bool(runs) and all(run.final_eval_status == EvalStatus.PASSED for run in runs) + score_values = [metric.score for metric in metrics if metric.score is not None] + score = sum(score_values) / len(score_values) if score_values else (1.0 if run_passed else 0.0) + actual = "" + key_trajectory = [] + if runs and runs[0].eval_metric_result_per_invocation: + actual_invocation = runs[0].eval_metric_result_per_invocation[-1].actual_invocation + actual = _text(actual_invocation.final_response) + for tool_call in get_all_tool_calls(actual_invocation.intermediate_data): + key_trajectory.append({ + "kind": "tool_call", + "name": tool_call.name or "", + "payload": { + "id": tool_call.id, + "args": tool_call.args or {} + }, + }) + for tool_response in get_all_tool_responses(actual_invocation.intermediate_data): + key_trajectory.append({ + "kind": "tool_response", + "name": tool_response.name or "", + "payload": { + "id": tool_response.id, + "response": tool_response.response + }, + }) + failure_reasons = (self._attribute_failure(case, runs, metric_templates) if not run_passed else []) + state = case.session_input.state if case.session_input else {} + return CaseEvaluation( + case_id=case.eval_id, + passed=run_passed, + score=score, + key_case=bool(state.get("key_case", False)), + hard_fail=bool(state.get("hard_fail", False)), + metrics=metrics, + primary_failure=failure_reasons[0].category if failure_reasons else None, + failure_reasons=failure_reasons, + actual_response=actual, + expected_response=self._expected_response(case), + key_trajectory=key_trajectory, + ) + + @staticmethod + def _expected_response(case: EvalCase) -> str: + conversation = case.conversation or [] + return _text(conversation[-1].final_response) if conversation else "" + + @staticmethod + def _attribute_failure( + case: EvalCase, + runs: list[EvalCaseResult], + metrics: dict[str, Any], + ) -> list[FailureReason]: + """Return evidence-backed reasons ordered from root cause to symptom.""" + if not runs or not runs[0].eval_metric_result_per_invocation: + return [ + FailureReason( + category="other_metric_failure", + explanation="AgentEvaluator produced no invocation result.", + evidence={"case_id": case.eval_id}, + ) + ] + invocation_result = runs[0].eval_metric_result_per_invocation[-1] + invocation_metrics = {metric.metric_name: metric for metric in invocation_result.eval_metric_results or []} + actual = invocation_result.actual_invocation + expected = invocation_result.expected_invocation + actual_calls = get_all_tool_calls(actual.intermediate_data) + expected_calls = get_all_tool_calls(expected.intermediate_data if expected else None) + actual_responses = get_all_tool_responses(actual.intermediate_data) + reasons: list[FailureReason] = [] + + all_metrics = dict(metrics) + all_metrics.update(invocation_metrics) + for metric_name, metric in all_metrics.items(): + if TraceEvaluator._metric_evaluated(metric): + continue + status = getattr(metric, "eval_status", EvalStatus.NOT_EVALUATED) + details = getattr(metric, "details", None) + reasons.append( + FailureReason( + category="evaluation_error", + explanation="An evaluator metric produced no usable verdict.", + evidence={ + "metric": metric_name, + "status": getattr(status, "name", str(status)), + "reason": details.reason if details and details.reason else "", + }, + )) + + error_responses = [] + for response in actual_responses: + payload = response.response + if isinstance(payload, dict) and (payload.get("error") + or str(payload.get("status", "")).lower() in {"error", "failed"}): + error_responses.append({ + "tool": response.name, + "id": response.id, + "response": payload, + }) + actual_names = [call.name for call in actual_calls] + expected_names = [call.name for call in expected_calls] + if error_responses: + reasons.append( + FailureReason( + category="tool_call_error", + explanation="A required tool returned an execution error.", + evidence={"tool_responses": error_responses}, + )) + elif actual_names != expected_names: + reasons.append( + FailureReason( + category="tool_call_error", + explanation="The actual tool selection does not match the expected trajectory.", + evidence={ + "actual_tools": actual_names, + "expected_tools": expected_names + }, + )) + + parameter_diffs = [] + if actual_names == expected_names: + for actual_call, expected_call in zip(actual_calls, expected_calls): + if (actual_call.args or {}) != (expected_call.args or {}): + parameter_diffs.append({ + "tool": actual_call.name, + "actual": actual_call.args or {}, + "expected": expected_call.args or {}, + }) + if parameter_diffs: + reasons.append( + FailureReason( + category="parameter_error", + explanation="A tool was selected correctly but received different arguments.", + evidence={"differences": parameter_diffs}, + )) + + rubric_metric = invocation_metrics.get("llm_rubric_response") or metrics.get("llm_rubric_response") + failed_rubrics = TraceEvaluator._failed_rubric_ids(rubric_metric) + rubric_failed = (TraceEvaluator._metric_evaluated(rubric_metric) + and not TraceEvaluator._metric_passed(rubric_metric)) + rubric_scores_available = bool(rubric_metric and rubric_metric.details and rubric_metric.details.rubric_scores) + format_rubric_status = TraceEvaluator._rubric_status(rubric_metric, "format") + evidence_base = { + "metric": "llm_rubric_response", + "score": getattr(rubric_metric, "score", None), + "threshold": getattr(rubric_metric, "threshold", None), + "judge_reason": + (rubric_metric.details.reason if rubric_metric is not None and rubric_metric.details else ""), + "rubric_scores_available": rubric_scores_available, + "failed_rubrics": failed_rubrics, + } + format_evidence: dict[str, Any] | None = None + if format_rubric_status is False: + format_evidence = {**evidence_base, "detector": "evaluator_rubric_id"} + elif format_rubric_status is None and rubric_failed and expected is not None: + # Some SDK aggregators omit per-rubric scores. Fall back to the + # deterministic replay verdict, then to explicit request syntax. + replay_signal, matched_variants = TraceEvaluator._replay_format_signal(case, _text(actual.final_response)) + violations = TraceEvaluator._format_violations( + case, + expected=_text(expected.final_response), + actual=_text(actual.final_response), + ) + if replay_signal is False: + format_evidence = { + **evidence_base, + "detector": "replay_signal", + "format_pass": False, + "matched_variants": matched_variants, + "violations": violations, + } + elif replay_signal is None: + if violations: + format_evidence = { + **evidence_base, + "detector": "requested_format", + "violations": violations, + } + format_failed = format_evidence is not None + if format_evidence is not None: + reasons.append( + FailureReason( + category="format_failure", + explanation="The format rubric failed for the final response.", + evidence=format_evidence, + )) + + knowledge_metric = invocation_metrics.get("llm_rubric_knowledge_recall") or metrics.get( + "llm_rubric_knowledge_recall") + if (TraceEvaluator._metric_evaluated(knowledge_metric) and not TraceEvaluator._metric_passed(knowledge_metric)): + reasons.append( + FailureReason( + category="knowledge_recall_insufficiency", + explanation="Retrieved knowledge did not cover the facts required by the rubric.", + evidence={ + "metric": "llm_rubric_knowledge_recall", + "score": knowledge_metric.score, + "threshold": knowledge_metric.threshold, + }, + )) + + quality_rubrics = [rubric_id for rubric_id in failed_rubrics if "format" not in rubric_id] + if rubric_failed and not format_failed: + reasons.append( + FailureReason( + category="llm_rubric_failure", + explanation="The final answer failed one or more non-format quality rubrics.", + evidence={ + "failed_rubrics": quality_rubrics, + "score": rubric_metric.score, + "threshold": rubric_metric.threshold, + }, + )) + + final_metric = metrics.get("final_response_avg_score") + if TraceEvaluator._metric_evaluated(final_metric) and not TraceEvaluator._metric_passed(final_metric): + reasons.append( + FailureReason( + category="final_response_mismatch", + explanation="The final response does not contain the expected reference answer.", + evidence={ + "actual": _text(actual.final_response), + "expected": _text(expected.final_response) if expected else "", + }, + )) + + trajectory_metric = metrics.get("tool_trajectory_avg_score") + if (TraceEvaluator._metric_evaluated(trajectory_metric) and not TraceEvaluator._metric_passed(trajectory_metric) + and not any(reason.category in {"tool_call_error", "parameter_error"} for reason in reasons)): + reasons.append( + FailureReason( + category="tool_call_error", + explanation="The tool trajectory metric failed without a narrower argument diagnosis.", + evidence={ + "score": trajectory_metric.score, + "threshold": trajectory_metric.threshold, + }, + )) + if not reasons: + failed_metrics = [name for name, metric in metrics.items() if not TraceEvaluator._metric_passed(metric)] + reasons.append( + FailureReason( + category="other_metric_failure", + explanation="At least one evaluator metric failed without a specialized diagnosis.", + evidence={"failed_metrics": failed_metrics}, + )) + return reasons + + @staticmethod + def _metric_passed(metric: Any) -> bool: + return metric.score is not None and float(metric.score) >= float(metric.threshold) + + @staticmethod + def _metric_evaluated(metric: Any) -> bool: + return (metric is not None and metric.score is not None + and getattr(metric, "eval_status", EvalStatus.NOT_EVALUATED) != EvalStatus.NOT_EVALUATED) + + @staticmethod + def _failed_rubric_ids(metric: Any) -> list[str]: + if metric is None or metric.details is None: + return [] + failed: list[str] = [] + for rubric in metric.details.rubric_scores or []: + rubric_id = str(getattr(rubric, "id", "") or "").lower() + score = getattr(rubric, "score", None) + if score is not None and float(score) < 1.0: + failed.append(rubric_id) + return failed + + @staticmethod + def _rubric_status(metric: Any, rubric_name: str) -> bool | None: + if metric is None or metric.details is None: + return None + matching_scores = [] + for rubric in metric.details.rubric_scores or []: + rubric_id = str(getattr(rubric, "id", "") or "").casefold() + if rubric_name.casefold() not in rubric_id: + continue + score = getattr(rubric, "score", None) + if score is not None: + matching_scores.append(float(score)) + return all(score >= 1.0 for score in matching_scores) if matching_scores else None + + @staticmethod + def _replay_format_signal(case: EvalCase, actual: str) -> tuple[bool | None, list[str]]: + """Return an unambiguous offline format verdict for the replayed answer.""" + state = case.session_input.state if case.session_input else {} + matches: dict[str, bool] = {} + for variant, payload in (state.get("variant_traces") or {}).items(): + if not isinstance(payload, dict) or _payload_response_text(payload) != actual: + continue + signals = payload.get("signals") or {} + value = signals.get("format_pass") + if isinstance(value, bool): + matches[str(variant)] = value + verdicts = set(matches.values()) + verdict = next(iter(verdicts)) if len(verdicts) == 1 else None + return verdict, sorted(matches) + + @staticmethod + def _format_violations( + case: EvalCase, + *, + expected: str, + actual: str, + ) -> list[str]: + """Detect deterministic format mismatches requested by the evaluation case.""" + conversation = case.conversation or [] + request = _text(conversation[0].user_content).casefold() if conversation else "" + violations: list[str] = [] + + json_requested = TraceEvaluator._format_requested( + request, + positive_pattern=(r"(?:(?:只|仅)?(?:返回|输出|回复|回答|使用|采用|用|格式(?:为)?).{0,16}json|" + r"json.{0,12}(?:格式|对象|数组|返回|输出|回复|回答)|" + r"(?:return|output|respond|reply|use|provide).{0,16}json)"), + format_pattern=r"json", + ) + if json_requested: + try: + json.loads(actual) + except (TypeError, ValueError, json.JSONDecodeError): + violations.append("invalid_json") + + single_line_pattern = r"(?:一行|单行|single[ -]?line|one[ -]?line)" + single_line_requested = TraceEvaluator._format_requested( + request, + positive_pattern=(r"(?:(?:只|仅)?(?:返回|输出|回复|回答).{0,12}(?:一行|单行)|" + r"(?:一行|单行).{0,8}(?:格式|输出|返回|回复)|" + r"(?:return|output|respond|reply).{0,12}(?:single|one)[ -]?line|" + r"(?:single|one)[ -]?line.{0,12}(?:format|output|response))"), + format_pattern=single_line_pattern, + ) + if single_line_requested and len(actual.splitlines()) != 1: + violations.append("not_single_line") + + markdown_requested = TraceEvaluator._format_requested( + request, + positive_pattern=(r"(?:(?:返回|输出|回复|回答|使用|采用|用|格式(?:为)?).{0,16}markdown|" + r"markdown.{0,12}(?:格式|表格|列表|返回|输出|回复|回答)|" + r"(?:return|output|respond|reply|use|provide).{0,16}markdown)"), + format_pattern=r"markdown", + ) + if markdown_requested: + missing = TraceEvaluator._markdown_structures(expected) - TraceEvaluator._markdown_structures(actual) + violations.extend(f"missing_markdown_{item}" for item in sorted(missing)) + return violations + + @staticmethod + def _format_requested( + request: str, + *, + positive_pattern: str, + format_pattern: str, + ) -> bool: + clauses = [item.strip() for item in re.split(r"[，。；;,.!?！？\n]+", request) if item.strip()] + return any( + re.search(positive_pattern, clause, re.IGNORECASE) + and not TraceEvaluator._format_request_negated(clause, format_pattern) for clause in clauses) + + @staticmethod + def _format_request_negated(clause: str, format_pattern: str) -> bool: + chinese_prefix = r"(?:不要|无需|不必|不应|禁止|避免|请勿|切勿|不得|不能|不可)" + chinese_action = r"(?:只|直接|再)?(?:返回|输出|回复|回答|使用|采用|用|以)" + english_action = r"(?:return|output|respond|reply|use|provide)" + patterns = [ + rf"{chinese_prefix}{chinese_action}.{{0,8}}{format_pattern}", + rf"{chinese_prefix}\s*{format_pattern}", + rf"(?:不用|不使用|不采用|不返回|不输出|不回复|不回答).{{0,8}}{format_pattern}", + rf"(?:不是|非)\s*{format_pattern}(?:\s*格式)?", + rf"(?:do\s+not|don't|not(?!\s+only)|never)\s+{english_action}.{{0,12}}{format_pattern}", + rf"(?:without|avoid|no|never)\s+(?:{english_action}\s+)?{format_pattern}", + rf"(?:not|no)\s+{format_pattern}(?:\s+format)?", + ] + return any(re.search(pattern, clause, re.IGNORECASE) for pattern in patterns) + + @staticmethod + def _markdown_structures(text: str) -> set[str]: + structures: set[str] = set() + if re.search(r"(?m)^#{1,6}\s", text): + structures.add("heading") + if re.search(r"(?m)^(?:[-*+]\s|\d+[.)]\s)", text): + structures.add("list") + if re.search(r"(?m)^\s*\|?(?:\s*:?-{3,}:?\s*\|)+\s*$", text): + structures.add("table") + if "```" in text: + structures.add("fence") + return structures diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 00000000..6296b0c6 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,81 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "contains", + "case_insensitive": false + } + } + } + }, + { + "metric_name": "llm_rubric_response", + "threshold": 1.0, + "criterion": { + "llm_judge": { + "judge_model": { + "provider_name": "offline", + "model_name": "judge", + "api_key": "", + "num_samples": 1, + "generation_config": { + "max_tokens": 512, + "temperature": 0.0 + } + }, + "rubrics": [ + { + "id": "format", + "content": { + "text": "The response follows the required output format." + }, + "description": "Required output format", + "type": "FINAL_RESPONSE_QUALITY" + }, + { + "id": "quality", + "content": { + "text": "The response gives the complete, ordered, and useful answer." + }, + "description": "Complete and useful response", + "type": "FINAL_RESPONSE_QUALITY" + } + ] + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 1, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 91, + "reflection_lm": { + "provider_name": "offline", + "model_name": "reflector", + "api_key": "", + "generation_config": { + "max_tokens": 1024, + "temperature": 0.0 + } + }, + "candidate_selection_strategy": "current_best", + "module_selector": "round_robin", + "reflection_minibatch_size": 3, + "skip_perfect_score": false, + "max_metric_calls": 80, + "max_candidate_proposals": 3, + "timeout_seconds": 120 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline.json b/examples/optimization/eval_optimize_loop/pipeline.json new file mode 100644 index 00000000..98cfdb7f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.json @@ -0,0 +1,29 @@ +{ + "optimizer_config": "optimizer.json", + "regression_metrics_config": "regression.metrics.json", + "gate_config": "gate.json", + "train_dataset": "data/train.evalset.json", + "validation_dataset": "data/val.evalset.json", + "target_prompts": { + "system": "agent/prompts/system.md" + }, + "candidate_sources": [ + { + "candidate_id": "ineffective", + "path": "agent/prompts/candidates/ineffective.md" + }, + { + "candidate_id": "overfit", + "path": "agent/prompts/candidates/overfit.md" + }, + { + "candidate_id": "robust", + "path": "agent/prompts/candidates/robust.md" + } + ], + "output_dir": "runs/latest", + "seed": 91, + "bootstrap_samples": 2000, + "confidence_level": 0.95, + "apply_if_accepted": false +} diff --git a/examples/optimization/eval_optimize_loop/regression.metrics.json b/examples/optimization/eval_optimize_loop/regression.metrics.json new file mode 100644 index 00000000..72b92513 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/regression.metrics.json @@ -0,0 +1,81 @@ +{ + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "threshold": 1.0, + "criterion": { + "tool_trajectory": { + "default": { + "name": {"match": "exact", "case_insensitive": false}, + "arguments": {"match": "exact"} + }, + "order_sensitive": true, + "subset_matching": false + } + } + }, + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": {"match": "contains", "case_insensitive": false} + } + } + }, + { + "metric_name": "llm_rubric_response", + "threshold": 1.0, + "criterion": { + "llm_judge": { + "judge_model": { + "provider_name": "offline", + "model_name": "judge", + "api_key": "", + "num_samples": 1, + "generation_config": {"max_tokens": 512, "temperature": 0.0} + }, + "rubrics": [ + { + "id": "format", + "content": {"text": "The response follows the requested output format."}, + "description": "Format compliance", + "type": "FINAL_RESPONSE_QUALITY" + }, + { + "id": "quality", + "content": {"text": "The response contains every required step and fact."}, + "description": "Answer completeness", + "type": "FINAL_RESPONSE_QUALITY" + } + ] + } + } + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "threshold": 1.0, + "criterion": { + "llm_judge": { + "judge_model": { + "provider_name": "offline", + "model_name": "judge", + "api_key": "", + "num_samples": 1, + "generation_config": {"max_tokens": 512, "temperature": 0.0} + }, + "rubrics": [ + { + "id": "knowledge_recall", + "content": {"text": "Retrieved knowledge contains the facts required by the question."}, + "description": "Knowledge coverage", + "type": "KNOWLEDGE_COVERAGE" + } + ], + "knowledge_tool_names": ["search_policy"] + } + } + } + ], + "num_runs": 1 +} diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..36fb975b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,77 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""One-command entry point for the evaluation/optimization regression loop.""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from datetime import datetime +from datetime import timezone +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[2] +_OPTIMIZATION_EXAMPLES = _HERE.parent +for path in (_REPO_ROOT, _OPTIMIZATION_EXAMPLES): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) + +from eval_optimize_loop.loop import EvalOptimizePipeline +from eval_optimize_loop.loop import PipelineSpec + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=("Run baseline evaluation, real GEPA prompt optimization, " + "independent regression, gate, and reports without an API key."), ) + parser.add_argument( + "--manifest", + type=Path, + default=_HERE / "pipeline.json", + help="Pipeline manifest (default: pipeline.json beside this script).", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=None, + help="Artifact directory. Default: runs/.", + ) + parser.add_argument( + "--apply-if-accepted", + action="store_true", + help="Atomically write the selected prompt back only after every gate passes.", + ) + return parser + + +async def _run(args: argparse.Namespace) -> int: + output_dir = args.output_dir + if output_dir is None: + stamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ") + output_dir = _HERE / "runs" / stamp + spec = PipelineSpec.from_file( + args.manifest, + output_dir=output_dir, + ) + if args.apply_if_accepted: + spec = spec.model_copy(update={"apply_if_accepted": True}) + report = await EvalOptimizePipeline(spec).run() + print(f"decision={report.status}") + print(f"selected_candidate={report.selected_candidate_id or 'none'}") + print(f"json_report={spec.output_dir / 'optimization_report.json'}") + print(f"markdown_report={spec.output_dir / 'optimization_report.md'}") + return 0 if report.status == "accepted" else 2 + + +def main() -> int: + return asyncio.run(_run(_parser().parse_args())) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json new file mode 100644 index 00000000..06eb35c1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -0,0 +1,3401 @@ +{ + "schema_version": "1.0", + "status": "accepted", + "baseline": { + "train": { + "split": "train", + "pass_rate": 0.0, + "average_score": 0.6666666666666666, + "cases": [ + { + "case_id": "train_final_response_mismatch", + "passed": false, + "score": 0.75, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": "final_response_mismatch", + "failure_reasons": [ + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "工单 T-100 状态：处理中。", + "expected": "工单 T-100 状态：已完成。" + } + } + ], + "actual_response": "工单 T-100 状态：处理中。", + "expected_response": "工单 T-100 状态：已完成。", + "key_trajectory": [] + }, + { + "case_id": "train_tool_call_error", + "passed": false, + "score": 0.75, + "key_case": false, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": "tool_call_error", + "failure_reasons": [ + { + "category": "tool_call_error", + "explanation": "A required tool returned an execution error.", + "evidence": { + "tool_responses": [ + { + "tool": "lookup_order", + "id": "call-1", + "response": { + "error": "backend unavailable" + } + } + ] + } + }, + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "无法查询订单状态。", + "expected": "订单 O-200 状态：已发货。" + } + } + ], + "actual_response": "无法查询订单状态。", + "expected_response": "订单 O-200 状态：已发货。", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-200" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "error": "backend unavailable" + } + } + } + ] + }, + { + "case_id": "train_parameter_error", + "passed": false, + "score": 0.5, + "key_case": false, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": "parameter_error", + "failure_reasons": [ + { + "category": "parameter_error", + "explanation": "A tool was selected correctly but received different arguments.", + "evidence": { + "differences": [ + { + "tool": "lookup_order", + "actual": { + "order_id": "O-399" + }, + "expected": { + "order_id": "O-300" + } + } + ] + } + }, + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "订单 O-399 状态：已取消。", + "expected": "订单 O-300 状态：处理中。" + } + } + ], + "actual_response": "订单 O-399 状态：已取消。", + "expected_response": "订单 O-300 状态：处理中。", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-399" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-399", + "status": "已取消" + } + } + } + ] + } + ] + }, + "validation": { + "split": "validation", + "pass_rate": 0.3333333333333333, + "average_score": 0.8333333333333334, + "cases": [ + { + "case_id": "val_llm_rubric_failure", + "passed": false, + "score": 0.875, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 0.5, + "threshold": 1.0, + "passed": false, + "reason": " model#0: score=0.5000 passed=False reason=replay evidence satisfies rubric replay evidence does not satisfy rubric\naggregator=all_pass -> FAILED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": "llm_rubric_failure", + "failure_reasons": [ + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "actual_response": "退款处理流程：请告知客户耐心等待。", + "expected_response": "退款处理流程：", + "key_trajectory": [] + }, + { + "case_id": "val_knowledge_recall_insufficiency", + "passed": false, + "score": 0.625, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 0.5, + "threshold": 1.0, + "passed": false, + "reason": " model#0: score=0.5000 passed=False reason=replay evidence satisfies rubric replay evidence does not satisfy rubric\naggregator=all_pass -> FAILED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": " model#0: score=0.0000 passed=False reason=replay evidence does not satisfy rubric\naggregator=all_pass -> FAILED" + } + ], + "primary_failure": "knowledge_recall_insufficiency", + "failure_reasons": [ + { + "category": "knowledge_recall_insufficiency", + "explanation": "Retrieved knowledge did not cover the facts required by the rubric.", + "evidence": { + "metric": "llm_rubric_knowledge_recall", + "score": 0.0, + "threshold": 1.0 + } + }, + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "actual_response": "会员退款到账：通常当天到账。", + "expected_response": "会员退款到账：", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "search_policy", + "payload": { + "id": "call-1", + "args": { + "query": "会员退款到账时效", + "top_k": 2 + } + } + }, + { + "kind": "tool_response", + "name": "search_policy", + "payload": { + "id": "call-1", + "response": { + "documents": [ + { + "id": "return-window", + "text": "商品支持 7 天退换。" + } + ] + } + } + } + ] + }, + { + "case_id": "val_format_key_case", + "passed": true, + "score": 1.0, + "key_case": true, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}", + "expected_response": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-400" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-400", + "status": "已签收" + } + } + } + ] + } + ] + } + }, + "candidates": [ + { + "candidate_id": "ineffective", + "prompts": { + "system": "[variant: ineffective]\n\nYou are a helpful customer-support agent. Be friendly and concise." + }, + "train": { + "split": "train", + "pass_rate": 0.0, + "average_score": 0.6666666666666666, + "cases": [ + { + "case_id": "train_final_response_mismatch", + "passed": false, + "score": 0.75, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": "final_response_mismatch", + "failure_reasons": [ + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "工单 T-100 状态：处理中。", + "expected": "工单 T-100 状态：已完成。" + } + } + ], + "actual_response": "工单 T-100 状态：处理中。", + "expected_response": "工单 T-100 状态：已完成。", + "key_trajectory": [] + }, + { + "case_id": "train_tool_call_error", + "passed": false, + "score": 0.75, + "key_case": false, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": "tool_call_error", + "failure_reasons": [ + { + "category": "tool_call_error", + "explanation": "A required tool returned an execution error.", + "evidence": { + "tool_responses": [ + { + "tool": "lookup_order", + "id": "call-1", + "response": { + "error": "backend unavailable" + } + } + ] + } + }, + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "无法查询订单状态。", + "expected": "订单 O-200 状态：已发货。" + } + } + ], + "actual_response": "无法查询订单状态。", + "expected_response": "订单 O-200 状态：已发货。", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-200" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "error": "backend unavailable" + } + } + } + ] + }, + { + "case_id": "train_parameter_error", + "passed": false, + "score": 0.5, + "key_case": false, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": "parameter_error", + "failure_reasons": [ + { + "category": "parameter_error", + "explanation": "A tool was selected correctly but received different arguments.", + "evidence": { + "differences": [ + { + "tool": "lookup_order", + "actual": { + "order_id": "O-399" + }, + "expected": { + "order_id": "O-300" + } + } + ] + } + }, + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "订单 O-399 状态：已取消。", + "expected": "订单 O-300 状态：处理中。" + } + } + ], + "actual_response": "订单 O-399 状态：已取消。", + "expected_response": "订单 O-300 状态：处理中。", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-399" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-399", + "status": "已取消" + } + } + } + ] + } + ] + }, + "validation": { + "split": "validation", + "pass_rate": 0.3333333333333333, + "average_score": 0.8333333333333334, + "cases": [ + { + "case_id": "val_llm_rubric_failure", + "passed": false, + "score": 0.875, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 0.5, + "threshold": 1.0, + "passed": false, + "reason": " model#0: score=0.5000 passed=False reason=replay evidence satisfies rubric replay evidence does not satisfy rubric\naggregator=all_pass -> FAILED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": "llm_rubric_failure", + "failure_reasons": [ + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "actual_response": "退款处理流程：请告知客户耐心等待。", + "expected_response": "退款处理流程：", + "key_trajectory": [] + }, + { + "case_id": "val_knowledge_recall_insufficiency", + "passed": false, + "score": 0.625, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 0.5, + "threshold": 1.0, + "passed": false, + "reason": " model#0: score=0.5000 passed=False reason=replay evidence satisfies rubric replay evidence does not satisfy rubric\naggregator=all_pass -> FAILED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": " model#0: score=0.0000 passed=False reason=replay evidence does not satisfy rubric\naggregator=all_pass -> FAILED" + } + ], + "primary_failure": "knowledge_recall_insufficiency", + "failure_reasons": [ + { + "category": "knowledge_recall_insufficiency", + "explanation": "Retrieved knowledge did not cover the facts required by the rubric.", + "evidence": { + "metric": "llm_rubric_knowledge_recall", + "score": 0.0, + "threshold": 1.0 + } + }, + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "actual_response": "会员退款到账：通常当天到账。", + "expected_response": "会员退款到账：", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "search_policy", + "payload": { + "id": "call-1", + "args": { + "query": "会员退款到账时效", + "top_k": 2 + } + } + }, + { + "kind": "tool_response", + "name": "search_policy", + "payload": { + "id": "call-1", + "response": { + "documents": [ + { + "id": "return-window", + "text": "商品支持 7 天退换。" + } + ] + } + } + } + ] + }, + { + "case_id": "val_format_key_case", + "passed": true, + "score": 1.0, + "key_case": true, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}", + "expected_response": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-400" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-400", + "status": "已签收" + } + } + } + ] + } + ] + }, + "delta": { + "train": { + "split": "train", + "pass_rate_delta": 0.0, + "average_score_delta": 0.0, + "paired_pass_rate_ci": { + "point_estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "bootstrap_samples": 2000, + "seed": 91 + }, + "newly_passed": [], + "newly_failed": [], + "score_improved": [], + "score_regressed": [], + "unchanged": [ + "train_final_response_mismatch", + "train_parameter_error", + "train_tool_call_error" + ], + "cases": [ + { + "case_id": "train_final_response_mismatch", + "status": "unchanged", + "baseline_passed": false, + "candidate_passed": false, + "baseline_score": 0.75, + "candidate_score": 0.75, + "score_delta": 0.0 + }, + { + "case_id": "train_parameter_error", + "status": "unchanged", + "baseline_passed": false, + "candidate_passed": false, + "baseline_score": 0.5, + "candidate_score": 0.5, + "score_delta": 0.0 + }, + { + "case_id": "train_tool_call_error", + "status": "unchanged", + "baseline_passed": false, + "candidate_passed": false, + "baseline_score": 0.75, + "candidate_score": 0.75, + "score_delta": 0.0 + } + ] + }, + "validation": { + "split": "validation", + "pass_rate_delta": 0.0, + "average_score_delta": 0.0, + "paired_pass_rate_ci": { + "point_estimate": 0.0, + "lower": 0.0, + "upper": 0.0, + "confidence_level": 0.95, + "bootstrap_samples": 2000, + "seed": 91 + }, + "newly_passed": [], + "newly_failed": [], + "score_improved": [], + "score_regressed": [], + "unchanged": [ + "val_format_key_case", + "val_knowledge_recall_insufficiency", + "val_llm_rubric_failure" + ], + "cases": [ + { + "case_id": "val_format_key_case", + "status": "unchanged", + "baseline_passed": true, + "candidate_passed": true, + "baseline_score": 1.0, + "candidate_score": 1.0, + "score_delta": 0.0 + }, + { + "case_id": "val_knowledge_recall_insufficiency", + "status": "unchanged", + "baseline_passed": false, + "candidate_passed": false, + "baseline_score": 0.625, + "candidate_score": 0.625, + "score_delta": 0.0 + }, + { + "case_id": "val_llm_rubric_failure", + "status": "unchanged", + "baseline_passed": false, + "candidate_passed": false, + "baseline_score": 0.875, + "candidate_score": 0.875, + "score_delta": 0.0 + } + ] + } + }, + "gate": { + "accepted": false, + "overfitting_detected": false, + "checks": [ + { + "name": "optimizer_succeeded", + "passed": true, + "actual": "SUCCEEDED", + "expected": "SUCCEEDED", + "reason": "AgentOptimizer must finish successfully.", + "required": true + }, + { + "name": "minimum_validation_gain", + "passed": false, + "actual": 0.0, + "expected": ">= 0.5", + "reason": "Candidate validation pass-rate gain must meet the configured floor.", + "required": true + }, + { + "name": "no_new_hard_failures", + "passed": true, + "actual": [], + "expected": [], + "reason": "A baseline pass may not become a hard failure.", + "required": true + }, + { + "name": "key_cases_no_regression", + "passed": true, + "actual": [], + "expected": [], + "reason": "Key validation cases may not lose score.", + "required": true + }, + { + "name": "no_train_validation_overfit", + "passed": true, + "actual": false, + "expected": false, + "reason": "Train improvement paired with validation regression is rejected; pass rate is primary and average score breaks ties.", + "required": true + }, + { + "name": "validation_gain_ci_lower_bound", + "passed": true, + "actual": 0.0, + "expected": ">= 0.0", + "reason": "The paired bootstrap lower bound must meet the configured confidence floor.", + "required": true + }, + { + "name": "metric_call_budget", + "passed": true, + "actual": 24, + "expected": "<= 80", + "reason": "Measured resource usage must stay within max_metric_calls.", + "required": true + }, + { + "name": "token_budget", + "passed": true, + "actual": 368, + "expected": "<= 200000", + "reason": "Measured resource usage must stay within max_total_tokens.", + "required": true + }, + { + "name": "duration_budget", + "passed": true, + "actual": 0.009248999878764153, + "expected": "<= 180", + "reason": "Measured resource usage must stay within max_duration_seconds.", + "required": true + }, + { + "name": "cost_budget", + "passed": true, + "actual": 0.0, + "expected": "<= 0.0", + "reason": "Unknown cost fails closed; a measured cost must be within budget.", + "required": true + } + ] + }, + "audit": { + "prompt_sha256": "88f3ba2d74e39464b4f8423aa87eacd13b9ebe64e0109851e4d65d8dffac9405", + "source": "GEPA on_proposal_end", + "optimizer_round": 1, + "seed": 91, + "resources": { + "metric_calls": 24, + "reflection_calls": 0, + "judge_calls": 12, + "prompt_tokens": 270, + "completion_tokens": 98, + "total_tokens": 368, + "cost_usd": 0.0, + "duration_seconds": 0.009248999878764153, + "p95_latency_ms": 8.0, + "cost_measurement": "measured_zero_offline" + } + }, + "pareto_optimal": false + }, + { + "candidate_id": "overfit", + "prompts": { + "system": "[variant: overfit]\n\nYou are a customer-support agent tuned to the three training examples. Memorize their answers and prefer them over general rules." + }, + "train": { + "split": "train", + "pass_rate": 1.0, + "average_score": 1.0, + "cases": [ + { + "case_id": "train_final_response_mismatch", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "工单 T-100 状态：已完成。", + "expected_response": "工单 T-100 状态：已完成。", + "key_trajectory": [] + }, + { + "case_id": "train_tool_call_error", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "订单 O-200 状态：已发货。", + "expected_response": "订单 O-200 状态：已发货。", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-200" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-200", + "status": "已发货" + } + } + } + ] + }, + { + "case_id": "train_parameter_error", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "订单 O-300 状态：处理中。", + "expected_response": "订单 O-300 状态：处理中。", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-300" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-300", + "status": "处理中" + } + } + } + ] + } + ] + }, + "validation": { + "split": "validation", + "pass_rate": 0.0, + "average_score": 0.7083333333333334, + "cases": [ + { + "case_id": "val_llm_rubric_failure", + "passed": false, + "score": 0.875, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 0.5, + "threshold": 1.0, + "passed": false, + "reason": " model#0: score=0.5000 passed=False reason=replay evidence satisfies rubric replay evidence does not satisfy rubric\naggregator=all_pass -> FAILED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": "llm_rubric_failure", + "failure_reasons": [ + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "actual_response": "退款处理流程：请告知客户耐心等待。", + "expected_response": "退款处理流程：", + "key_trajectory": [] + }, + { + "case_id": "val_knowledge_recall_insufficiency", + "passed": false, + "score": 0.625, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 0.5, + "threshold": 1.0, + "passed": false, + "reason": " model#0: score=0.5000 passed=False reason=replay evidence satisfies rubric replay evidence does not satisfy rubric\naggregator=all_pass -> FAILED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": " model#0: score=0.0000 passed=False reason=replay evidence does not satisfy rubric\naggregator=all_pass -> FAILED" + } + ], + "primary_failure": "knowledge_recall_insufficiency", + "failure_reasons": [ + { + "category": "knowledge_recall_insufficiency", + "explanation": "Retrieved knowledge did not cover the facts required by the rubric.", + "evidence": { + "metric": "llm_rubric_knowledge_recall", + "score": 0.0, + "threshold": 1.0 + } + }, + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "actual_response": "会员退款到账：通常当天到账。", + "expected_response": "会员退款到账：", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "search_policy", + "payload": { + "id": "call-1", + "args": { + "query": "会员退款到账时效", + "top_k": 2 + } + } + }, + { + "kind": "tool_response", + "name": "search_policy", + "payload": { + "id": "call-1", + "response": { + "documents": [ + { + "id": "return-window", + "text": "商品支持 7 天退换。" + } + ] + } + } + } + ] + }, + { + "case_id": "val_format_key_case", + "passed": false, + "score": 0.625, + "key_case": true, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 0.0, + "threshold": 1.0, + "passed": false, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 0.5, + "threshold": 1.0, + "passed": false, + "reason": " model#0: score=0.5000 passed=False reason=replay evidence does not satisfy rubric replay evidence satisfies rubric\naggregator=all_pass -> FAILED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": "format_failure", + "failure_reasons": [ + { + "category": "format_failure", + "explanation": "The format rubric failed for the final response.", + "evidence": { + "metric": "llm_rubric_response", + "score": 0.5, + "threshold": 1.0, + "judge_reason": " model#0: score=0.5000 passed=False reason=replay evidence does not satisfy rubric replay evidence satisfies rubric\naggregator=all_pass -> FAILED", + "rubric_scores_available": false, + "failed_rubrics": [], + "detector": "replay_signal", + "format_pass": false, + "matched_variants": [ + "overfit" + ], + "violations": [ + "invalid_json" + ] + } + }, + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "订单 O-400 状态：已签收。", + "expected": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}" + } + } + ], + "actual_response": "订单 O-400 状态：已签收。", + "expected_response": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-400" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-400", + "status": "已签收" + } + } + } + ] + } + ] + }, + "delta": { + "train": { + "split": "train", + "pass_rate_delta": 1.0, + "average_score_delta": 0.33333333333333337, + "paired_pass_rate_ci": { + "point_estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "bootstrap_samples": 2000, + "seed": 91 + }, + "newly_passed": [ + "train_final_response_mismatch", + "train_parameter_error", + "train_tool_call_error" + ], + "newly_failed": [], + "score_improved": [], + "score_regressed": [], + "unchanged": [], + "cases": [ + { + "case_id": "train_final_response_mismatch", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.75, + "candidate_score": 1.0, + "score_delta": 0.25 + }, + { + "case_id": "train_parameter_error", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.5, + "candidate_score": 1.0, + "score_delta": 0.5 + }, + { + "case_id": "train_tool_call_error", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.75, + "candidate_score": 1.0, + "score_delta": 0.25 + } + ] + }, + "validation": { + "split": "validation", + "pass_rate_delta": -0.3333333333333333, + "average_score_delta": -0.125, + "paired_pass_rate_ci": { + "point_estimate": -0.3333333333333333, + "lower": -1.0, + "upper": 0.0, + "confidence_level": 0.95, + "bootstrap_samples": 2000, + "seed": 91 + }, + "newly_passed": [], + "newly_failed": [ + "val_format_key_case" + ], + "score_improved": [], + "score_regressed": [], + "unchanged": [ + "val_knowledge_recall_insufficiency", + "val_llm_rubric_failure" + ], + "cases": [ + { + "case_id": "val_format_key_case", + "status": "newly_failed", + "baseline_passed": true, + "candidate_passed": false, + "baseline_score": 1.0, + "candidate_score": 0.625, + "score_delta": -0.375 + }, + { + "case_id": "val_knowledge_recall_insufficiency", + "status": "unchanged", + "baseline_passed": false, + "candidate_passed": false, + "baseline_score": 0.625, + "candidate_score": 0.625, + "score_delta": 0.0 + }, + { + "case_id": "val_llm_rubric_failure", + "status": "unchanged", + "baseline_passed": false, + "candidate_passed": false, + "baseline_score": 0.875, + "candidate_score": 0.875, + "score_delta": 0.0 + } + ] + } + }, + "gate": { + "accepted": false, + "overfitting_detected": true, + "checks": [ + { + "name": "optimizer_succeeded", + "passed": true, + "actual": "SUCCEEDED", + "expected": "SUCCEEDED", + "reason": "AgentOptimizer must finish successfully.", + "required": true + }, + { + "name": "minimum_validation_gain", + "passed": false, + "actual": -0.3333333333333333, + "expected": ">= 0.5", + "reason": "Candidate validation pass-rate gain must meet the configured floor.", + "required": true + }, + { + "name": "no_new_hard_failures", + "passed": false, + "actual": [ + "val_format_key_case" + ], + "expected": [], + "reason": "A baseline pass may not become a hard failure.", + "required": true + }, + { + "name": "key_cases_no_regression", + "passed": false, + "actual": [ + "val_format_key_case" + ], + "expected": [], + "reason": "Key validation cases may not lose score.", + "required": true + }, + { + "name": "no_train_validation_overfit", + "passed": false, + "actual": true, + "expected": false, + "reason": "Train improvement paired with validation regression is rejected; pass rate is primary and average score breaks ties.", + "required": true + }, + { + "name": "validation_gain_ci_lower_bound", + "passed": false, + "actual": -1.0, + "expected": ">= 0.0", + "reason": "The paired bootstrap lower bound must meet the configured confidence floor.", + "required": true + }, + { + "name": "metric_call_budget", + "passed": true, + "actual": 24, + "expected": "<= 80", + "reason": "Measured resource usage must stay within max_metric_calls.", + "required": true + }, + { + "name": "token_budget", + "passed": true, + "actual": 392, + "expected": "<= 200000", + "reason": "Measured resource usage must stay within max_total_tokens.", + "required": true + }, + { + "name": "duration_budget", + "passed": true, + "actual": 0.010194707894697785, + "expected": "<= 180", + "reason": "Measured resource usage must stay within max_duration_seconds.", + "required": true + }, + { + "name": "cost_budget", + "passed": true, + "actual": 0.0, + "expected": "<= 0.0", + "reason": "Unknown cost fails closed; a measured cost must be within budget.", + "required": true + } + ] + }, + "audit": { + "prompt_sha256": "9db289952527d74f31bffae07c7545644361f178a72b69b026122eabb45096ec", + "source": "GEPA on_proposal_end", + "optimizer_round": 2, + "seed": 91, + "resources": { + "metric_calls": 24, + "reflection_calls": 0, + "judge_calls": 12, + "prompt_tokens": 294, + "completion_tokens": 98, + "total_tokens": 392, + "cost_usd": 0.0, + "duration_seconds": 0.010194707894697785, + "p95_latency_ms": 8.0, + "cost_measurement": "measured_zero_offline" + } + }, + "pareto_optimal": false + }, + { + "candidate_id": "robust", + "prompts": { + "system": "[variant: robust]\n\nYou are a reliable customer-support agent. Select the correct tool, preserve exact identifiers and arguments, ground the answer in retrieved evidence, follow the requested format, and include every required step in order." + }, + "train": { + "split": "train", + "pass_rate": 1.0, + "average_score": 1.0, + "cases": [ + { + "case_id": "train_final_response_mismatch", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "工单 T-100 状态：已完成。", + "expected_response": "工单 T-100 状态：已完成。", + "key_trajectory": [] + }, + { + "case_id": "train_tool_call_error", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "订单 O-200 状态：已发货。", + "expected_response": "订单 O-200 状态：已发货。", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-200" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-200", + "status": "已发货" + } + } + } + ] + }, + { + "case_id": "train_parameter_error", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "订单 O-300 状态：处理中。", + "expected_response": "订单 O-300 状态：处理中。", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-300" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-300", + "status": "处理中" + } + } + } + ] + } + ] + }, + "validation": { + "split": "validation", + "pass_rate": 1.0, + "average_score": 1.0, + "cases": [ + { + "case_id": "val_llm_rubric_failure", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "退款处理流程：先核验订单与付款状态，再告知退款时效。", + "expected_response": "退款处理流程：", + "key_trajectory": [] + }, + { + "case_id": "val_knowledge_recall_insufficiency", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "会员退款到账：原路退回，3-5 个工作日到账。", + "expected_response": "会员退款到账：", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "search_policy", + "payload": { + "id": "call-1", + "args": { + "query": "会员退款到账时效", + "top_k": 2 + } + } + }, + { + "kind": "tool_response", + "name": "search_policy", + "payload": { + "id": "call-1", + "response": { + "documents": [ + { + "id": "refund-sla", + "text": "会员退款原路退回，3-5 个工作日到账。" + } + ] + } + } + } + ] + }, + { + "case_id": "val_format_key_case", + "passed": true, + "score": 1.0, + "key_case": true, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}", + "expected_response": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-400" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-400", + "status": "已签收" + } + } + } + ] + } + ] + }, + "delta": { + "train": { + "split": "train", + "pass_rate_delta": 1.0, + "average_score_delta": 0.33333333333333337, + "paired_pass_rate_ci": { + "point_estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "bootstrap_samples": 2000, + "seed": 91 + }, + "newly_passed": [ + "train_final_response_mismatch", + "train_parameter_error", + "train_tool_call_error" + ], + "newly_failed": [], + "score_improved": [], + "score_regressed": [], + "unchanged": [], + "cases": [ + { + "case_id": "train_final_response_mismatch", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.75, + "candidate_score": 1.0, + "score_delta": 0.25 + }, + { + "case_id": "train_parameter_error", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.5, + "candidate_score": 1.0, + "score_delta": 0.5 + }, + { + "case_id": "train_tool_call_error", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.75, + "candidate_score": 1.0, + "score_delta": 0.25 + } + ] + }, + "validation": { + "split": "validation", + "pass_rate_delta": 0.6666666666666667, + "average_score_delta": 0.16666666666666663, + "paired_pass_rate_ci": { + "point_estimate": 0.6666666666666666, + "lower": 0.0, + "upper": 1.0, + "confidence_level": 0.95, + "bootstrap_samples": 2000, + "seed": 91 + }, + "newly_passed": [ + "val_knowledge_recall_insufficiency", + "val_llm_rubric_failure" + ], + "newly_failed": [], + "score_improved": [], + "score_regressed": [], + "unchanged": [ + "val_format_key_case" + ], + "cases": [ + { + "case_id": "val_format_key_case", + "status": "unchanged", + "baseline_passed": true, + "candidate_passed": true, + "baseline_score": 1.0, + "candidate_score": 1.0, + "score_delta": 0.0 + }, + { + "case_id": "val_knowledge_recall_insufficiency", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.625, + "candidate_score": 1.0, + "score_delta": 0.375 + }, + { + "case_id": "val_llm_rubric_failure", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.875, + "candidate_score": 1.0, + "score_delta": 0.125 + } + ] + } + }, + "gate": { + "accepted": true, + "overfitting_detected": false, + "checks": [ + { + "name": "optimizer_succeeded", + "passed": true, + "actual": "SUCCEEDED", + "expected": "SUCCEEDED", + "reason": "AgentOptimizer must finish successfully.", + "required": true + }, + { + "name": "minimum_validation_gain", + "passed": true, + "actual": 0.6666666666666667, + "expected": ">= 0.5", + "reason": "Candidate validation pass-rate gain must meet the configured floor.", + "required": true + }, + { + "name": "no_new_hard_failures", + "passed": true, + "actual": [], + "expected": [], + "reason": "A baseline pass may not become a hard failure.", + "required": true + }, + { + "name": "key_cases_no_regression", + "passed": true, + "actual": [], + "expected": [], + "reason": "Key validation cases may not lose score.", + "required": true + }, + { + "name": "no_train_validation_overfit", + "passed": true, + "actual": false, + "expected": false, + "reason": "Train improvement paired with validation regression is rejected; pass rate is primary and average score breaks ties.", + "required": true + }, + { + "name": "validation_gain_ci_lower_bound", + "passed": true, + "actual": 0.0, + "expected": ">= 0.0", + "reason": "The paired bootstrap lower bound must meet the configured confidence floor.", + "required": true + }, + { + "name": "metric_call_budget", + "passed": true, + "actual": 24, + "expected": "<= 80", + "reason": "Measured resource usage must stay within max_metric_calls.", + "required": true + }, + { + "name": "token_budget", + "passed": true, + "actual": 444, + "expected": "<= 200000", + "reason": "Measured resource usage must stay within max_total_tokens.", + "required": true + }, + { + "name": "duration_budget", + "passed": true, + "actual": 0.010438166093081236, + "expected": "<= 180", + "reason": "Measured resource usage must stay within max_duration_seconds.", + "required": true + }, + { + "name": "cost_budget", + "passed": true, + "actual": 0.0, + "expected": "<= 0.0", + "reason": "Unknown cost fails closed; a measured cost must be within budget.", + "required": true + } + ] + }, + "audit": { + "prompt_sha256": "61df0f34831108fda47d32c49932c49dc276c6708d9d1eaf999f216e4c109f26", + "source": "GEPA on_proposal_end", + "optimizer_round": 3, + "seed": 91, + "resources": { + "metric_calls": 24, + "reflection_calls": 0, + "judge_calls": 12, + "prompt_tokens": 330, + "completion_tokens": 114, + "total_tokens": 444, + "cost_usd": 0.0, + "duration_seconds": 0.010438166093081236, + "p95_latency_ms": 8.0, + "cost_measurement": "measured_zero_offline" + } + }, + "pareto_optimal": true + } + ], + "selected_candidate_id": "robust", + "candidate": { + "candidate_id": "robust", + "prompts": { + "system": "[variant: robust]\n\nYou are a reliable customer-support agent. Select the correct tool, preserve exact identifiers and arguments, ground the answer in retrieved evidence, follow the requested format, and include every required step in order." + }, + "train": { + "split": "train", + "pass_rate": 1.0, + "average_score": 1.0, + "cases": [ + { + "case_id": "train_final_response_mismatch", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "工单 T-100 状态：已完成。", + "expected_response": "工单 T-100 状态：已完成。", + "key_trajectory": [] + }, + { + "case_id": "train_tool_call_error", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "订单 O-200 状态：已发货。", + "expected_response": "订单 O-200 状态：已发货。", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-200" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-200", + "status": "已发货" + } + } + } + ] + }, + { + "case_id": "train_parameter_error", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "订单 O-300 状态：处理中。", + "expected_response": "订单 O-300 状态：处理中。", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-300" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-300", + "status": "处理中" + } + } + } + ] + } + ] + }, + "validation": { + "split": "validation", + "pass_rate": 1.0, + "average_score": 1.0, + "cases": [ + { + "case_id": "val_llm_rubric_failure", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "退款处理流程：先核验订单与付款状态，再告知退款时效。", + "expected_response": "退款处理流程：", + "key_trajectory": [] + }, + { + "case_id": "val_knowledge_recall_insufficiency", + "passed": true, + "score": 1.0, + "key_case": false, + "hard_fail": false, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "会员退款到账：原路退回，3-5 个工作日到账。", + "expected_response": "会员退款到账：", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "search_policy", + "payload": { + "id": "call-1", + "args": { + "query": "会员退款到账时效", + "top_k": 2 + } + } + }, + { + "kind": "tool_response", + "name": "search_policy", + "payload": { + "id": "call-1", + "response": { + "documents": [ + { + "id": "refund-sla", + "text": "会员退款原路退回，3-5 个工作日到账。" + } + ] + } + } + } + ] + }, + { + "case_id": "val_format_key_case", + "passed": true, + "score": 1.0, + "key_case": true, + "hard_fail": true, + "metrics": [ + { + "metric_name": "tool_trajectory_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "final_response_avg_score", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": "" + }, + { + "metric_name": "llm_rubric_response", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + }, + { + "metric_name": "llm_rubric_knowledge_recall", + "score": 1.0, + "threshold": 1.0, + "passed": true, + "reason": " model#0: score=1.0000 passed=True reason=replay evidence satisfies rubric\naggregator=all_pass -> PASSED" + } + ], + "primary_failure": null, + "failure_reasons": [], + "actual_response": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}", + "expected_response": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}", + "key_trajectory": [ + { + "kind": "tool_call", + "name": "lookup_order", + "payload": { + "id": "call-1", + "args": { + "order_id": "O-400" + } + } + }, + { + "kind": "tool_response", + "name": "lookup_order", + "payload": { + "id": "call-1", + "response": { + "order_id": "O-400", + "status": "已签收" + } + } + } + ] + } + ] + }, + "delta": { + "train": { + "split": "train", + "pass_rate_delta": 1.0, + "average_score_delta": 0.33333333333333337, + "paired_pass_rate_ci": { + "point_estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "bootstrap_samples": 2000, + "seed": 91 + }, + "newly_passed": [ + "train_final_response_mismatch", + "train_parameter_error", + "train_tool_call_error" + ], + "newly_failed": [], + "score_improved": [], + "score_regressed": [], + "unchanged": [], + "cases": [ + { + "case_id": "train_final_response_mismatch", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.75, + "candidate_score": 1.0, + "score_delta": 0.25 + }, + { + "case_id": "train_parameter_error", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.5, + "candidate_score": 1.0, + "score_delta": 0.5 + }, + { + "case_id": "train_tool_call_error", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.75, + "candidate_score": 1.0, + "score_delta": 0.25 + } + ] + }, + "validation": { + "split": "validation", + "pass_rate_delta": 0.6666666666666667, + "average_score_delta": 0.16666666666666663, + "paired_pass_rate_ci": { + "point_estimate": 0.6666666666666666, + "lower": 0.0, + "upper": 1.0, + "confidence_level": 0.95, + "bootstrap_samples": 2000, + "seed": 91 + }, + "newly_passed": [ + "val_knowledge_recall_insufficiency", + "val_llm_rubric_failure" + ], + "newly_failed": [], + "score_improved": [], + "score_regressed": [], + "unchanged": [ + "val_format_key_case" + ], + "cases": [ + { + "case_id": "val_format_key_case", + "status": "unchanged", + "baseline_passed": true, + "candidate_passed": true, + "baseline_score": 1.0, + "candidate_score": 1.0, + "score_delta": 0.0 + }, + { + "case_id": "val_knowledge_recall_insufficiency", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.625, + "candidate_score": 1.0, + "score_delta": 0.375 + }, + { + "case_id": "val_llm_rubric_failure", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.875, + "candidate_score": 1.0, + "score_delta": 0.125 + } + ] + } + }, + "gate": { + "accepted": true, + "overfitting_detected": false, + "checks": [ + { + "name": "optimizer_succeeded", + "passed": true, + "actual": "SUCCEEDED", + "expected": "SUCCEEDED", + "reason": "AgentOptimizer must finish successfully.", + "required": true + }, + { + "name": "minimum_validation_gain", + "passed": true, + "actual": 0.6666666666666667, + "expected": ">= 0.5", + "reason": "Candidate validation pass-rate gain must meet the configured floor.", + "required": true + }, + { + "name": "no_new_hard_failures", + "passed": true, + "actual": [], + "expected": [], + "reason": "A baseline pass may not become a hard failure.", + "required": true + }, + { + "name": "key_cases_no_regression", + "passed": true, + "actual": [], + "expected": [], + "reason": "Key validation cases may not lose score.", + "required": true + }, + { + "name": "no_train_validation_overfit", + "passed": true, + "actual": false, + "expected": false, + "reason": "Train improvement paired with validation regression is rejected; pass rate is primary and average score breaks ties.", + "required": true + }, + { + "name": "validation_gain_ci_lower_bound", + "passed": true, + "actual": 0.0, + "expected": ">= 0.0", + "reason": "The paired bootstrap lower bound must meet the configured confidence floor.", + "required": true + }, + { + "name": "metric_call_budget", + "passed": true, + "actual": 24, + "expected": "<= 80", + "reason": "Measured resource usage must stay within max_metric_calls.", + "required": true + }, + { + "name": "token_budget", + "passed": true, + "actual": 444, + "expected": "<= 200000", + "reason": "Measured resource usage must stay within max_total_tokens.", + "required": true + }, + { + "name": "duration_budget", + "passed": true, + "actual": 0.010438166093081236, + "expected": "<= 180", + "reason": "Measured resource usage must stay within max_duration_seconds.", + "required": true + }, + { + "name": "cost_budget", + "passed": true, + "actual": 0.0, + "expected": "<= 0.0", + "reason": "Unknown cost fails closed; a measured cost must be within budget.", + "required": true + } + ] + }, + "audit": { + "prompt_sha256": "61df0f34831108fda47d32c49932c49dc276c6708d9d1eaf999f216e4c109f26", + "source": "GEPA on_proposal_end", + "optimizer_round": 3, + "seed": 91, + "resources": { + "metric_calls": 24, + "reflection_calls": 0, + "judge_calls": 12, + "prompt_tokens": 330, + "completion_tokens": 114, + "total_tokens": 444, + "cost_usd": 0.0, + "duration_seconds": 0.010438166093081236, + "p95_latency_ms": 8.0, + "cost_measurement": "measured_zero_offline" + } + }, + "pareto_optimal": true + }, + "delta": { + "train": { + "split": "train", + "pass_rate_delta": 1.0, + "average_score_delta": 0.33333333333333337, + "paired_pass_rate_ci": { + "point_estimate": 1.0, + "lower": 1.0, + "upper": 1.0, + "confidence_level": 0.95, + "bootstrap_samples": 2000, + "seed": 91 + }, + "newly_passed": [ + "train_final_response_mismatch", + "train_parameter_error", + "train_tool_call_error" + ], + "newly_failed": [], + "score_improved": [], + "score_regressed": [], + "unchanged": [], + "cases": [ + { + "case_id": "train_final_response_mismatch", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.75, + "candidate_score": 1.0, + "score_delta": 0.25 + }, + { + "case_id": "train_parameter_error", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.5, + "candidate_score": 1.0, + "score_delta": 0.5 + }, + { + "case_id": "train_tool_call_error", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.75, + "candidate_score": 1.0, + "score_delta": 0.25 + } + ] + }, + "validation": { + "split": "validation", + "pass_rate_delta": 0.6666666666666667, + "average_score_delta": 0.16666666666666663, + "paired_pass_rate_ci": { + "point_estimate": 0.6666666666666666, + "lower": 0.0, + "upper": 1.0, + "confidence_level": 0.95, + "bootstrap_samples": 2000, + "seed": 91 + }, + "newly_passed": [ + "val_knowledge_recall_insufficiency", + "val_llm_rubric_failure" + ], + "newly_failed": [], + "score_improved": [], + "score_regressed": [], + "unchanged": [ + "val_format_key_case" + ], + "cases": [ + { + "case_id": "val_format_key_case", + "status": "unchanged", + "baseline_passed": true, + "candidate_passed": true, + "baseline_score": 1.0, + "candidate_score": 1.0, + "score_delta": 0.0 + }, + { + "case_id": "val_knowledge_recall_insufficiency", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.625, + "candidate_score": 1.0, + "score_delta": 0.375 + }, + { + "case_id": "val_llm_rubric_failure", + "status": "newly_passed", + "baseline_passed": false, + "candidate_passed": true, + "baseline_score": 0.875, + "candidate_score": 1.0, + "score_delta": 0.125 + } + ] + } + }, + "gate": { + "accepted": true, + "overfitting_detected": false, + "checks": [ + { + "name": "optimizer_succeeded", + "passed": true, + "actual": "SUCCEEDED", + "expected": "SUCCEEDED", + "reason": "AgentOptimizer must finish successfully.", + "required": true + }, + { + "name": "minimum_validation_gain", + "passed": true, + "actual": 0.6666666666666667, + "expected": ">= 0.5", + "reason": "Candidate validation pass-rate gain must meet the configured floor.", + "required": true + }, + { + "name": "no_new_hard_failures", + "passed": true, + "actual": [], + "expected": [], + "reason": "A baseline pass may not become a hard failure.", + "required": true + }, + { + "name": "key_cases_no_regression", + "passed": true, + "actual": [], + "expected": [], + "reason": "Key validation cases may not lose score.", + "required": true + }, + { + "name": "no_train_validation_overfit", + "passed": true, + "actual": false, + "expected": false, + "reason": "Train improvement paired with validation regression is rejected; pass rate is primary and average score breaks ties.", + "required": true + }, + { + "name": "validation_gain_ci_lower_bound", + "passed": true, + "actual": 0.0, + "expected": ">= 0.0", + "reason": "The paired bootstrap lower bound must meet the configured confidence floor.", + "required": true + }, + { + "name": "metric_call_budget", + "passed": true, + "actual": 24, + "expected": "<= 80", + "reason": "Measured resource usage must stay within max_metric_calls.", + "required": true + }, + { + "name": "token_budget", + "passed": true, + "actual": 444, + "expected": "<= 200000", + "reason": "Measured resource usage must stay within max_total_tokens.", + "required": true + }, + { + "name": "duration_budget", + "passed": true, + "actual": 0.010438166093081236, + "expected": "<= 180", + "reason": "Measured resource usage must stay within max_duration_seconds.", + "required": true + }, + { + "name": "cost_budget", + "passed": true, + "actual": 0.0, + "expected": "<= 0.0", + "reason": "Unknown cost fails closed; a measured cost must be within budget.", + "required": true + } + ] + }, + "failure_attribution": { + "explained_failed_cases": 13, + "total_failed_cases": 13, + "coverage_rate": 1.0, + "category_counts": { + "final_response_mismatch": 7, + "format_failure": 1, + "knowledge_recall_insufficiency": 3, + "llm_rubric_failure": 6, + "parameter_error": 2, + "tool_call_error": 2 + }, + "by_case": { + "baseline/train/train_final_response_mismatch": [ + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "工单 T-100 状态：处理中。", + "expected": "工单 T-100 状态：已完成。" + } + } + ], + "baseline/train/train_tool_call_error": [ + { + "category": "tool_call_error", + "explanation": "A required tool returned an execution error.", + "evidence": { + "tool_responses": [ + { + "tool": "lookup_order", + "id": "call-1", + "response": { + "error": "backend unavailable" + } + } + ] + } + }, + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "无法查询订单状态。", + "expected": "订单 O-200 状态：已发货。" + } + } + ], + "baseline/train/train_parameter_error": [ + { + "category": "parameter_error", + "explanation": "A tool was selected correctly but received different arguments.", + "evidence": { + "differences": [ + { + "tool": "lookup_order", + "actual": { + "order_id": "O-399" + }, + "expected": { + "order_id": "O-300" + } + } + ] + } + }, + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "订单 O-399 状态：已取消。", + "expected": "订单 O-300 状态：处理中。" + } + } + ], + "baseline/validation/val_llm_rubric_failure": [ + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "baseline/validation/val_knowledge_recall_insufficiency": [ + { + "category": "knowledge_recall_insufficiency", + "explanation": "Retrieved knowledge did not cover the facts required by the rubric.", + "evidence": { + "metric": "llm_rubric_knowledge_recall", + "score": 0.0, + "threshold": 1.0 + } + }, + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "candidate/ineffective/train/train_final_response_mismatch": [ + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "工单 T-100 状态：处理中。", + "expected": "工单 T-100 状态：已完成。" + } + } + ], + "candidate/ineffective/train/train_tool_call_error": [ + { + "category": "tool_call_error", + "explanation": "A required tool returned an execution error.", + "evidence": { + "tool_responses": [ + { + "tool": "lookup_order", + "id": "call-1", + "response": { + "error": "backend unavailable" + } + } + ] + } + }, + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "无法查询订单状态。", + "expected": "订单 O-200 状态：已发货。" + } + } + ], + "candidate/ineffective/train/train_parameter_error": [ + { + "category": "parameter_error", + "explanation": "A tool was selected correctly but received different arguments.", + "evidence": { + "differences": [ + { + "tool": "lookup_order", + "actual": { + "order_id": "O-399" + }, + "expected": { + "order_id": "O-300" + } + } + ] + } + }, + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "订单 O-399 状态：已取消。", + "expected": "订单 O-300 状态：处理中。" + } + } + ], + "candidate/ineffective/validation/val_llm_rubric_failure": [ + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "candidate/ineffective/validation/val_knowledge_recall_insufficiency": [ + { + "category": "knowledge_recall_insufficiency", + "explanation": "Retrieved knowledge did not cover the facts required by the rubric.", + "evidence": { + "metric": "llm_rubric_knowledge_recall", + "score": 0.0, + "threshold": 1.0 + } + }, + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "candidate/overfit/validation/val_llm_rubric_failure": [ + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "candidate/overfit/validation/val_knowledge_recall_insufficiency": [ + { + "category": "knowledge_recall_insufficiency", + "explanation": "Retrieved knowledge did not cover the facts required by the rubric.", + "evidence": { + "metric": "llm_rubric_knowledge_recall", + "score": 0.0, + "threshold": 1.0 + } + }, + { + "category": "llm_rubric_failure", + "explanation": "The final answer failed one or more non-format quality rubrics.", + "evidence": { + "failed_rubrics": [], + "score": 0.5, + "threshold": 1.0 + } + } + ], + "candidate/overfit/validation/val_format_key_case": [ + { + "category": "format_failure", + "explanation": "The format rubric failed for the final response.", + "evidence": { + "metric": "llm_rubric_response", + "score": 0.5, + "threshold": 1.0, + "judge_reason": " model#0: score=0.5000 passed=False reason=replay evidence does not satisfy rubric replay evidence satisfies rubric\naggregator=all_pass -> FAILED", + "rubric_scores_available": false, + "failed_rubrics": [], + "detector": "replay_signal", + "format_pass": false, + "matched_variants": [ + "overfit" + ], + "violations": [ + "invalid_json" + ] + } + }, + { + "category": "final_response_mismatch", + "explanation": "The final response does not contain the expected reference answer.", + "evidence": { + "actual": "订单 O-400 状态：已签收。", + "expected": "{\"order_id\":\"O-400\",\"status\":\"已签收\"}" + } + } + ] + } + }, + "optimizer": { + "algorithm": "gepa_reflective", + "status": "SUCCEEDED", + "stop_reason": "max_candidate_proposals", + "used_agent_optimizer": true, + "baseline_pass_rate": 0.8333333333333334, + "best_pass_rate": 1.0, + "rounds": 3, + "resources": { + "metric_calls": 27, + "reflection_calls": 3, + "judge_calls": null, + "prompt_tokens": 7054, + "completion_tokens": 125, + "total_tokens": 7179, + "cost_usd": 0.0, + "duration_seconds": 0.055892, + "p95_latency_ms": null, + "cost_measurement": "measured_zero_offline" + }, + "artifact_dir": "optimizer" + }, + "data_quality": { + "passed": true, + "train_cases": 3, + "validation_cases": 3, + "duplicate_ids": [], + "cross_split_duplicates": [], + "near_cross_split_duplicates": [], + "prompt_leakage_matches": [] + }, + "audit": { + "run_id": "offline-87974874280c", + "started_at": "2026-07-11T11:24:37.877369+00:00", + "finished_at": "2026-07-11T11:24:37.976539+00:00", + "duration_seconds": 0.09917154186405241, + "seed": 91, + "config_sha256": "489e04411cee9728eb1374d454ea2821a9e92a7c0f8ea478ca2d9cad4d6675d2", + "train_sha256": "4ea1e92e8c6bb77c6e770fc5e7b546264c956213df89b731c0a421c3635c33ae", + "validation_sha256": "da4dd12bc0e83462b86e83ad5f59841a3d73ac1b4942e068f97981fd486fd4ff", + "baseline_prompt_sha256": { + "system": "7c9707da7fcae322da42a4ff26e9d7635de1e15d5337cfc7d1e1291d887134ba" + }, + "command": "python examples/optimization/eval_optimize_loop/run_pipeline.py" + } +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md new file mode 100644 index 00000000..bd80bf6a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md @@ -0,0 +1,123 @@ +# Evaluation + Optimization Regression Report + +- **Decision:** `accepted` +- **Selected candidate:** `robust` +- **Optimizer:** `gepa_reflective` / `SUCCEEDED` +- **Seed:** `91` + +## Baseline + +| Split | Pass rate | Average score | Cases | +| --- | ---: | ---: | ---: | +| Train | 0.000 | 0.667 | 3 | +| Validation | 0.333 | 0.833 | 3 | + +## Candidate matrix + +| Candidate | Train pass | Validation pass | Validation delta | Paired CI | Gate | Overfit | Pareto | +| --- | ---: | ---: | ---: | --- | --- | --- | --- | +| `ineffective` | 0.000 | 0.333 | +0.000 | [+0.000, +0.000] | FAIL | False | False | +| `overfit` | 1.000 | 0.000 | -0.333 | [-1.000, +0.000] | FAIL | True | False | +| `robust` | 1.000 | 1.000 | +0.667 | [+0.000, +1.000] | PASS | False | True | + +## Candidate + +Candidate `robust` was independently re-evaluated. + +| Split | Baseline | Candidate | Delta | +| --- | ---: | ---: | ---: | +| Train pass rate | 0.000 | 1.000 | +1.000 | +| Validation pass rate | 0.333 | 1.000 | +0.667 | + +Paired bootstrap interval: `[+0.000, +1.000]` at 95% confidence. + +### Per-case delta + +| Case | Split | Status | Score delta | +| --- | --- | --- | ---: | +| `train_final_response_mismatch` | train | newly_passed | +0.250 | +| `train_parameter_error` | train | newly_passed | +0.500 | +| `train_tool_call_error` | train | newly_passed | +0.250 | +| `val_format_key_case` | validation | unchanged | +0.000 | +| `val_knowledge_recall_insufficiency` | validation | newly_passed | +0.375 | +| `val_llm_rubric_failure` | validation | newly_passed | +0.125 | + +## Gate + +| Candidate | Check | Required | Result | Actual | Expected | +| --- | --- | --- | --- | --- | --- | +| `ineffective` | `optimizer_succeeded` | True | PASS | `SUCCEEDED` | `SUCCEEDED` | +| `ineffective` | `minimum_validation_gain` | True | FAIL | `0.0` | `>= 0.5` | +| `ineffective` | `no_new_hard_failures` | True | PASS | `[]` | `[]` | +| `ineffective` | `key_cases_no_regression` | True | PASS | `[]` | `[]` | +| `ineffective` | `no_train_validation_overfit` | True | PASS | `False` | `False` | +| `ineffective` | `validation_gain_ci_lower_bound` | True | PASS | `0.0` | `>= 0.0` | +| `ineffective` | `metric_call_budget` | True | PASS | `24` | `<= 80` | +| `ineffective` | `token_budget` | True | PASS | `368` | `<= 200000` | +| `ineffective` | `duration_budget` | True | PASS | `0.009248999878764153` | `<= 180` | +| `ineffective` | `cost_budget` | True | PASS | `0.0` | `<= 0.0` | +| `overfit` | `optimizer_succeeded` | True | PASS | `SUCCEEDED` | `SUCCEEDED` | +| `overfit` | `minimum_validation_gain` | True | FAIL | `-0.3333333333333333` | `>= 0.5` | +| `overfit` | `no_new_hard_failures` | True | FAIL | `['val_format_key_case']` | `[]` | +| `overfit` | `key_cases_no_regression` | True | FAIL | `['val_format_key_case']` | `[]` | +| `overfit` | `no_train_validation_overfit` | True | FAIL | `True` | `False` | +| `overfit` | `validation_gain_ci_lower_bound` | True | FAIL | `-1.0` | `>= 0.0` | +| `overfit` | `metric_call_budget` | True | PASS | `24` | `<= 80` | +| `overfit` | `token_budget` | True | PASS | `392` | `<= 200000` | +| `overfit` | `duration_budget` | True | PASS | `0.010194707894697785` | `<= 180` | +| `overfit` | `cost_budget` | True | PASS | `0.0` | `<= 0.0` | +| `robust` | `optimizer_succeeded` | True | PASS | `SUCCEEDED` | `SUCCEEDED` | +| `robust` | `minimum_validation_gain` | True | PASS | `0.6666666666666667` | `>= 0.5` | +| `robust` | `no_new_hard_failures` | True | PASS | `[]` | `[]` | +| `robust` | `key_cases_no_regression` | True | PASS | `[]` | `[]` | +| `robust` | `no_train_validation_overfit` | True | PASS | `False` | `False` | +| `robust` | `validation_gain_ci_lower_bound` | True | PASS | `0.0` | `>= 0.0` | +| `robust` | `metric_call_budget` | True | PASS | `24` | `<= 80` | +| `robust` | `token_budget` | True | PASS | `444` | `<= 200000` | +| `robust` | `duration_budget` | True | PASS | `0.010438166093081236` | `<= 180` | +| `robust` | `cost_budget` | True | PASS | `0.0` | `<= 0.0` | + +## Failure attribution + +Explained 13/13 failed cases (100.0%). + +| Case | Category | Explanation | +| --- | --- | --- | +| `baseline/train/train_final_response_mismatch` | `final_response_mismatch` | The final response does not contain the expected reference answer. | +| `baseline/train/train_tool_call_error` | `tool_call_error` | A required tool returned an execution error. | +| `baseline/train/train_tool_call_error` | `final_response_mismatch` | The final response does not contain the expected reference answer. | +| `baseline/train/train_parameter_error` | `parameter_error` | A tool was selected correctly but received different arguments. | +| `baseline/train/train_parameter_error` | `final_response_mismatch` | The final response does not contain the expected reference answer. | +| `baseline/validation/val_llm_rubric_failure` | `llm_rubric_failure` | The final answer failed one or more non-format quality rubrics. | +| `baseline/validation/val_knowledge_recall_insufficiency` | `knowledge_recall_insufficiency` | Retrieved knowledge did not cover the facts required by the rubric. | +| `baseline/validation/val_knowledge_recall_insufficiency` | `llm_rubric_failure` | The final answer failed one or more non-format quality rubrics. | +| `candidate/ineffective/train/train_final_response_mismatch` | `final_response_mismatch` | The final response does not contain the expected reference answer. | +| `candidate/ineffective/train/train_tool_call_error` | `tool_call_error` | A required tool returned an execution error. | +| `candidate/ineffective/train/train_tool_call_error` | `final_response_mismatch` | The final response does not contain the expected reference answer. | +| `candidate/ineffective/train/train_parameter_error` | `parameter_error` | A tool was selected correctly but received different arguments. | +| `candidate/ineffective/train/train_parameter_error` | `final_response_mismatch` | The final response does not contain the expected reference answer. | +| `candidate/ineffective/validation/val_llm_rubric_failure` | `llm_rubric_failure` | The final answer failed one or more non-format quality rubrics. | +| `candidate/ineffective/validation/val_knowledge_recall_insufficiency` | `knowledge_recall_insufficiency` | Retrieved knowledge did not cover the facts required by the rubric. | +| `candidate/ineffective/validation/val_knowledge_recall_insufficiency` | `llm_rubric_failure` | The final answer failed one or more non-format quality rubrics. | +| `candidate/overfit/validation/val_llm_rubric_failure` | `llm_rubric_failure` | The final answer failed one or more non-format quality rubrics. | +| `candidate/overfit/validation/val_knowledge_recall_insufficiency` | `knowledge_recall_insufficiency` | Retrieved knowledge did not cover the facts required by the rubric. | +| `candidate/overfit/validation/val_knowledge_recall_insufficiency` | `llm_rubric_failure` | The final answer failed one or more non-format quality rubrics. | +| `candidate/overfit/validation/val_format_key_case` | `format_failure` | The format rubric failed for the final response. | +| `candidate/overfit/validation/val_format_key_case` | `final_response_mismatch` | The final response does not contain the expected reference answer. | + +## Audit + +- Run id: `offline-87974874280c` +- Duration: `0.099s` +- Metric calls: `27` +- Tokens: `7179` +- Cost measurement: `measured_zero_offline` +- Optimizer artifacts: `optimizer` + +### Candidate evaluation resources + +| Candidate | Metric calls | Judge calls | Tokens | P95 latency | Duration | Cost | Cost measurement | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| `ineffective` | 24 | 12 | 368 | 8.0 ms | 0.009 s | $0.0000 | `measured_zero_offline` | +| `overfit` | 24 | 12 | 392 | 8.0 ms | 0.010 s | $0.0000 | `measured_zero_offline` | +| `robust` | 24 | 12 | 444 | 8.0 ms | 0.010 s | $0.0000 | `measured_zero_offline` | diff --git a/tests/evaluation/test_eval_optimize_loop_example.py b/tests/evaluation/test_eval_optimize_loop_example.py new file mode 100644 index 00000000..dadc45f0 --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_example.py @@ -0,0 +1,705 @@ +# +# Tencent is pleased to support the open source community by making trpc-agent-python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# trpc-agent-python is licensed under the Apache License Version 2.0. +# +"""Public-interface tests for the evaluation/optimization regression loop example.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +_EXAMPLE_DIR = (Path(__file__).resolve().parents[2] / "examples" / "optimization" / "eval_optimize_loop") +_OPTIMIZATION_EXAMPLES_DIR = _EXAMPLE_DIR.parent + + +def _load_public_interface(): + if str(_OPTIMIZATION_EXAMPLES_DIR) not in sys.path: + sys.path.insert(0, str(_OPTIMIZATION_EXAMPLES_DIR)) + from eval_optimize_loop.loop import EvalOptimizePipeline + from eval_optimize_loop.loop import PipelineSpec + + return EvalOptimizePipeline, PipelineSpec + + +def test_example_deliverables_are_complete_and_sdk_configs_validate() -> None: + expected = { + "README.md", + "DESIGN.md", + "run_pipeline.py", + "pipeline.json", + "optimizer.json", + "regression.metrics.json", + "gate.json", + "agent/prompts/system.md", + "data/train.evalset.json", + "data/val.evalset.json", + "sample_output/optimization_report.json", + "sample_output/optimization_report.md", + } + assert all((_EXAMPLE_DIR / path).is_file() for path in expected) + + from trpc_agent_sdk.evaluation import EvalConfig + from trpc_agent_sdk.evaluation import EvalSet + from trpc_agent_sdk.evaluation._optimize_config import load_optimize_config + + load_optimize_config(str(_EXAMPLE_DIR / "optimizer.json")) + EvalConfig.model_validate_json((_EXAMPLE_DIR / "regression.metrics.json").read_text(encoding="utf-8")) + train = EvalSet.model_validate_json((_EXAMPLE_DIR / "data/train.evalset.json").read_text(encoding="utf-8")) + validation = EvalSet.model_validate_json((_EXAMPLE_DIR / "data/val.evalset.json").read_text(encoding="utf-8")) + assert len(train.eval_cases) == 3 + assert len(validation.eval_cases) == 3 + + design = (_EXAMPLE_DIR / "DESIGN.md").read_text(encoding="utf-8") + chinese_chars = sum("\u4e00" <= char <= "\u9fff" for char in design) + assert 300 <= chinese_chars <= 500 + + _load_public_interface() + from eval_optimize_loop.loop import OptimizationReport + + sample = OptimizationReport.model_validate_json( + (_EXAMPLE_DIR / "sample_output/optimization_report.json").read_text(encoding="utf-8")) + assert sample.selected_candidate_id == "robust" + + +def test_output_override_is_resolved_from_callers_working_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, PipelineSpec = _load_public_interface() + monkeypatch.chdir(tmp_path) + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=Path("relative-run"), + ) + assert spec.output_dir == tmp_path / "relative-run" + + +@pytest.mark.asyncio +async def test_offline_pipeline_writes_machine_and_human_reports_without_api_key( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The one public seam runs the real SDK loop and returns its report.""" + for name in ( + "TRPC_AGENT_API_KEY", + "TRPC_AGENT_BASE_URL", + "TRPC_AGENT_MODEL_NAME", + "OPENAI_API_KEY", + ): + monkeypatch.delenv(name, raising=False) + + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + output_dir = tmp_path / "run" + prompt_path = _EXAMPLE_DIR / "agent/prompts/system.md" + original_prompt = prompt_path.read_bytes() + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=output_dir, + ) + + report = await EvalOptimizePipeline(spec).run() + + assert report.status == "accepted" + assert report.optimizer.algorithm == "gepa_reflective" + assert report.optimizer.used_agent_optimizer is True + assert report.selected_candidate_id is not None + assert prompt_path.read_bytes() == original_prompt + assert (output_dir / "optimization_report.json").is_file() + assert (output_dir / "optimization_report.md").is_file() + + payload = json.loads((output_dir / "optimization_report.json").read_text(encoding="utf-8")) + assert payload["status"] == "accepted" + assert payload["baseline"]["train"]["cases"] + assert payload["baseline"]["validation"]["cases"] + markdown = (output_dir / "optimization_report.md").read_text(encoding="utf-8") + assert "Baseline" in markdown + assert "Gate" in markdown + + +@pytest.mark.asyncio +async def test_every_unique_proposal_is_independently_re_evaluated_and_gated(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "all-candidates", + ) + + report = await EvalOptimizePipeline(spec).run() + + by_id = {candidate.candidate_id: candidate for candidate in report.candidates} + assert set(by_id) == {"ineffective", "overfit", "robust"} + + assert by_id["ineffective"].gate.accepted is False + assert by_id["ineffective"].delta.validation.pass_rate_delta == pytest.approx(0.0) + + assert by_id["overfit"].gate.accepted is False + assert by_id["overfit"].gate.overfitting_detected is True + assert by_id["overfit"].delta.train.pass_rate_delta > 0 + assert by_id["overfit"].delta.validation.pass_rate_delta < 0 + assert "val_format_key_case" in by_id["overfit"].delta.validation.newly_failed + + assert by_id["robust"].gate.accepted is True + assert report.selected_candidate_id == "robust" + assert report.candidate is not None + assert report.candidate.candidate_id == "robust" + + +@pytest.mark.asyncio +async def test_trace_replay_attributes_all_required_failure_categories(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + output_dir = tmp_path / "attribution" + report = await EvalOptimizePipeline(PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=output_dir, + )).run() + + cases = list(report.baseline.train.cases) + list(report.baseline.validation.cases) + for candidate in report.candidates: + cases.extend(candidate.train.cases) + cases.extend(candidate.validation.cases) + + observed = {case.primary_failure for case in cases if not case.passed and case.primary_failure is not None} + assert { + "final_response_mismatch", + "tool_call_error", + "parameter_error", + "llm_rubric_failure", + "knowledge_recall_insufficiency", + "format_failure", + }.issubset(observed) + assert all(case.failure_reasons for case in cases if not case.passed) + assert all(reason.explanation for case in cases for reason in case.failure_reasons) + assert report.failure_attribution.coverage_rate == pytest.approx(1.0) + assert { + "final_response_mismatch", + "tool_call_error", + "parameter_error", + "llm_rubric_failure", + "knowledge_recall_insufficiency", + "format_failure", + }.issubset(report.failure_attribution.category_counts) + + baseline_by_id = { + case.case_id: case.primary_failure + for case in report.baseline.train.cases + report.baseline.validation.cases if not case.passed + } + assert baseline_by_id == { + "train_final_response_mismatch": "final_response_mismatch", + "train_tool_call_error": "tool_call_error", + "train_parameter_error": "parameter_error", + "val_llm_rubric_failure": "llm_rubric_failure", + "val_knowledge_recall_insufficiency": "knowledge_recall_insufficiency", + } + overfit = next(item for item in report.candidates if item.candidate_id == "overfit") + format_case = next(item for item in overfit.validation.cases if item.case_id == "val_format_key_case") + assert format_case.primary_failure == "format_failure" + + implementation = "\n".join(path.read_text(encoding="utf-8") for path in (_EXAMPLE_DIR / "loop").glob("*.py")) + for case_id in set(baseline_by_id) | {"val_format_key_case"}: + assert case_id not in implementation + + trace_files = sorted((output_dir / "traces").glob("**/*.trace.evalset.json")) + assert len(trace_files) == 8 # baseline + three proposals, train + validation + knowledge_case = next(case for case in report.baseline.validation.cases + if case.case_id == "val_knowledge_recall_insufficiency") + assert "llm_rubric_knowledge_recall" in {metric.metric_name for metric in knowledge_case.metrics} + assert knowledge_case.key_trajectory + + +@pytest.mark.asyncio +async def test_report_includes_paired_bootstrap_uncertainty_and_pareto_selection(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + report = await EvalOptimizePipeline( + PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "statistics", + )).run() + + by_id = {candidate.candidate_id: candidate for candidate in report.candidates} + robust_ci = by_id["robust"].delta.validation.paired_pass_rate_ci + assert robust_ci.point_estimate == pytest.approx(2 / 3) + assert robust_ci.confidence_level == pytest.approx(0.95) + assert robust_ci.bootstrap_samples == 2000 + assert robust_ci.lower <= robust_ci.point_estimate <= robust_ci.upper + + overfit_ci = by_id["overfit"].delta.validation.paired_pass_rate_ci + assert overfit_ci.upper <= 0.0 + assert by_id["robust"].pareto_optimal is True + assert by_id["ineffective"].pareto_optimal is False + assert by_id["overfit"].pareto_optimal is False + + robust_checks = {check.name: check for check in by_id["robust"].gate.checks} + assert robust_checks["validation_gain_ci_lower_bound"].passed is True + + +@pytest.mark.asyncio +async def test_data_quality_fails_closed_on_cross_split_near_duplicate(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + train_payload = json.loads((_EXAMPLE_DIR / "data/train.evalset.json").read_text(encoding="utf-8")) + validation_payload = json.loads((_EXAMPLE_DIR / "data/val.evalset.json").read_text(encoding="utf-8")) + copied = json.loads(json.dumps(train_payload["eval_cases"][0]["conversation"])) + copied[0]["user_content"]["parts"][0]["text"] += " 请尽快。" + validation_payload["eval_cases"][0]["conversation"] = copied + contaminated = tmp_path / "near-duplicate.evalset.json" + contaminated.write_text( + json.dumps(validation_payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "rejected", + ).model_copy(update={"validation_dataset": contaminated}) + + with pytest.raises(ValueError, match="near_cross_split"): + await EvalOptimizePipeline(spec).run() + + +@pytest.mark.asyncio +async def test_candidate_audit_uses_independent_evaluation_resources(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + report = await EvalOptimizePipeline( + PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "candidate-audit", + )).run() + + for candidate in report.candidates: + usage = candidate.audit.resources + assert usage.metric_calls == 24 # 6 cases x 4 metrics x 1 run + assert usage.judge_calls == 12 # 6 cases x 2 offline judge metrics + assert usage.total_tokens > 0 + assert usage.duration_seconds > 0 + assert usage.cost_usd == pytest.approx(0.0) + assert usage.cost_measurement == "measured_zero_offline" + assert candidate.audit.seed == 91 + assert len(candidate.audit.prompt_sha256) == 64 + + +@pytest.mark.asyncio +async def test_regression_protections_can_be_explicitly_disabled(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + gate_payload = json.loads((_EXAMPLE_DIR / "gate.json").read_text(encoding="utf-8")) + gate_payload.update({ + "min_validation_gain": -1.0, + "min_validation_gain_ci_lower_bound": -1.0, + "forbid_new_hard_failures": False, + "key_cases_no_regression": False, + "reject_overfitting": False, + }) + permissive_gate = tmp_path / "permissive-gate.json" + permissive_gate.write_text( + json.dumps(gate_payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "permissive", + ).model_copy(update={"gate_config": permissive_gate}) + + report = await EvalOptimizePipeline(spec).run() + + overfit = next(candidate for candidate in report.candidates if candidate.candidate_id == "overfit") + assert overfit.gate.accepted is True + optional_checks = { + check.name: check + for check in overfit.gate.checks if check.name in { + "no_new_hard_failures", + "key_cases_no_regression", + "no_train_validation_overfit", + } + } + assert all(check.required is False for check in optional_checks.values()) + + +@pytest.mark.asyncio +async def test_score_only_train_improvement_and_validation_regression_is_overfit(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + train_payload = json.loads((_EXAMPLE_DIR / "data/train.evalset.json").read_text(encoding="utf-8")) + validation_payload = json.loads((_EXAMPLE_DIR / "data/val.evalset.json").read_text(encoding="utf-8")) + + train_case = next(case for case in train_payload["eval_cases"] if case["eval_id"] == "train_parameter_error") + train_case["session_input"]["state"]["variant_traces"]["ineffective"]["intermediate_data"] = json.loads( + json.dumps(train_case["conversation"][0]["intermediate_data"])) + + validation_case = next(case for case in validation_payload["eval_cases"] + if case["eval_id"] == "val_knowledge_recall_insufficiency") + validation_case["session_input"]["state"]["variant_traces"]["ineffective"]["intermediate_data"] = { + "tool_uses": [], + "tool_responses": [], + } + + train_path = tmp_path / "score-only-train.evalset.json" + validation_path = tmp_path / "score-only-validation.evalset.json" + train_path.write_text(json.dumps(train_payload, ensure_ascii=False, indent=2), encoding="utf-8") + validation_path.write_text(json.dumps(validation_payload, ensure_ascii=False, indent=2), encoding="utf-8") + + gate_payload = json.loads((_EXAMPLE_DIR / "gate.json").read_text(encoding="utf-8")) + gate_payload.update({ + "min_validation_gain": -1.0, + "min_validation_gain_ci_lower_bound": -1.0, + "forbid_new_hard_failures": False, + "key_cases_no_regression": False, + "reject_overfitting": True, + }) + gate_path = tmp_path / "score-only-gate.json" + gate_path.write_text(json.dumps(gate_payload, ensure_ascii=False, indent=2), encoding="utf-8") + + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "score-only", + ).model_copy(update={ + "train_dataset": train_path, + "validation_dataset": validation_path, + "gate_config": gate_path, + }) + report = await EvalOptimizePipeline(spec).run() + + ineffective = next(candidate for candidate in report.candidates if candidate.candidate_id == "ineffective") + assert ineffective.delta.train.pass_rate_delta == pytest.approx(0.0) + assert ineffective.delta.train.average_score_delta > 0 + assert ineffective.delta.validation.pass_rate_delta == pytest.approx(0.0) + assert ineffective.delta.validation.average_score_delta < 0 + assert ineffective.gate.overfitting_detected is True + assert ineffective.gate.accepted is False + + +@pytest.mark.asyncio +async def test_replay_cost_is_measured_and_enforced_by_gate(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + train_payload = json.loads((_EXAMPLE_DIR / "data/train.evalset.json").read_text(encoding="utf-8")) + validation_payload = json.loads((_EXAMPLE_DIR / "data/val.evalset.json").read_text(encoding="utf-8")) + for case in train_payload["eval_cases"] + validation_payload["eval_cases"]: + case["session_input"]["state"]["variant_traces"]["robust"]["usage"]["cost"] = 1.0 + + train_path = tmp_path / "cost-train.evalset.json" + validation_path = tmp_path / "cost-validation.evalset.json" + train_path.write_text(json.dumps(train_payload, ensure_ascii=False, indent=2), encoding="utf-8") + validation_path.write_text(json.dumps(validation_payload, ensure_ascii=False, indent=2), encoding="utf-8") + + gate_payload = json.loads((_EXAMPLE_DIR / "gate.json").read_text(encoding="utf-8")) + gate_payload["budget"]["max_cost_usd"] = 0.5 + gate_path = tmp_path / "cost-gate.json" + gate_path.write_text(json.dumps(gate_payload, ensure_ascii=False, indent=2), encoding="utf-8") + + metrics_payload = json.loads((_EXAMPLE_DIR / "regression.metrics.json").read_text(encoding="utf-8")) + metrics_payload["num_runs"] = 2 + metrics_path = tmp_path / "two-runs.metrics.json" + metrics_path.write_text(json.dumps(metrics_payload, ensure_ascii=False, indent=2), encoding="utf-8") + + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "cost", + ).model_copy( + update={ + "train_dataset": train_path, + "validation_dataset": validation_path, + "gate_config": gate_path, + "regression_metrics_config": metrics_path, + }) + report = await EvalOptimizePipeline(spec).run() + + robust = next(candidate for candidate in report.candidates if candidate.candidate_id == "robust") + cost_check = next(check for check in robust.gate.checks if check.name == "cost_budget") + expected_single_run_tokens = sum(trace["usage"][key] + for case in train_payload["eval_cases"] + validation_payload["eval_cases"] + for trace in [case["session_input"]["state"]["variant_traces"]["robust"]] + for key in ("input_tokens", "output_tokens")) + assert robust.audit.resources.metric_calls == 48 + assert robust.audit.resources.judge_calls == 24 + assert robust.audit.resources.total_tokens == expected_single_run_tokens * 2 + assert robust.audit.resources.cost_usd == pytest.approx(12.0) + assert robust.audit.resources.cost_measurement == "measured_from_replay" + assert cost_check.passed is False + assert robust.gate.accepted is False + + +@pytest.mark.asyncio +async def test_plain_text_format_violation_is_attributed_without_json_heuristic(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + validation_payload = json.loads((_EXAMPLE_DIR / "data/val.evalset.json").read_text(encoding="utf-8")) + format_case = next(case for case in validation_payload["eval_cases"] if case["eval_id"] == "val_format_key_case") + format_case["conversation"][0]["user_content"]["parts"][0]["text"] = "查询订单 O-400，只返回一行纯文本，格式为“订单号|状态”。" + expected = "O-400|已签收" + format_case["conversation"][0]["final_response"]["parts"][0]["text"] = expected + traces = format_case["session_input"]["state"]["variant_traces"] + for variant in ("baseline", "ineffective", "robust"): + traces[variant]["final_response"]["parts"][0]["text"] = expected + for trace in traces.values(): + trace["signals"].pop("format_pass") + traces["overfit"]["final_response"]["parts"][0]["text"] = "订单号：O-400\n状态：已签收" + traces["overfit"]["signals"]["llm_rubric_pass"] = False + + validation_path = tmp_path / "plain-text-format.evalset.json" + validation_path.write_text( + json.dumps(validation_payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "plain-text-format", + ).model_copy(update={"validation_dataset": validation_path}) + + report = await EvalOptimizePipeline(spec).run() + + overfit = next(candidate for candidate in report.candidates if candidate.candidate_id == "overfit") + format_result = next(case for case in overfit.validation.cases if case.case_id == "val_format_key_case") + assert format_result.primary_failure == "format_failure" + evidence = format_result.failure_reasons[0].evidence + assert evidence["detector"] == "requested_format" + assert "not_single_line" in evidence["violations"] + + +@pytest.mark.asyncio +async def test_unknown_replay_cost_fails_closed_and_is_not_rendered_as_zero(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + train_payload = json.loads((_EXAMPLE_DIR / "data/train.evalset.json").read_text(encoding="utf-8")) + first_usage = train_payload["eval_cases"][0]["session_input"]["state"]["variant_traces"]["robust"]["usage"] + first_usage.pop("cost") + train_path = tmp_path / "unknown-cost.evalset.json" + train_path.write_text(json.dumps(train_payload, ensure_ascii=False, indent=2), encoding="utf-8") + + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "unknown-cost", + ).model_copy(update={"train_dataset": train_path}) + report = await EvalOptimizePipeline(spec).run() + + robust = next(candidate for candidate in report.candidates if candidate.candidate_id == "robust") + cost_check = next(check for check in robust.gate.checks if check.name == "cost_budget") + assert robust.audit.resources.cost_usd is None + assert robust.audit.resources.cost_measurement == "unavailable" + assert cost_check.actual == "unavailable" + assert cost_check.passed is False + markdown = (spec.output_dir / "optimization_report.md").read_text(encoding="utf-8") + robust_rows = [line for line in markdown.splitlines() if line.startswith("| `robust` |")] + resource_row = next(line for line in robust_rows if "unavailable" in line) + assert "$0.0000" not in resource_row + + +@pytest.mark.asyncio +async def test_json_shaped_reference_is_not_a_format_rule_without_a_json_request(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + validation_payload = json.loads((_EXAMPLE_DIR / "data/val.evalset.json").read_text(encoding="utf-8")) + quality_case = next(case for case in validation_payload["eval_cases"] + if case["eval_id"] == "val_llm_rubric_failure") + quality_case["conversation"][0]["user_content"]["parts"][0]["text"] = "完整说明退款流程；无需采用特定输出格式。" + quality_case["conversation"][0]["final_response"]["parts"][0]["text"] = '{"steps":["核验订单","告知时效"]}' + for trace in quality_case["session_input"]["state"]["variant_traces"].values(): + trace["signals"].pop("format_pass") + + validation_path = tmp_path / "json-shaped-reference.evalset.json" + validation_path.write_text( + json.dumps(validation_payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "json-shaped-reference", + ).model_copy(update={"validation_dataset": validation_path}) + report = await EvalOptimizePipeline(spec).run() + + quality_result = next(case for case in report.baseline.validation.cases if case.case_id == "val_llm_rubric_failure") + assert quality_result.primary_failure == "llm_rubric_failure" + + +def test_explicit_format_rubric_pass_is_authoritative_over_fallbacks() -> None: + _load_public_interface() + from eval_optimize_loop.loop.trace import TraceEvaluator + from trpc_agent_sdk.evaluation import EvalCase + from trpc_agent_sdk.evaluation import EvalCaseResult + from trpc_agent_sdk.evaluation import EvalMetricResult + from trpc_agent_sdk.evaluation import EvalMetricResultDetails + from trpc_agent_sdk.evaluation import EvalMetricResultPerInvocation + from trpc_agent_sdk.evaluation import EvalStatus + from trpc_agent_sdk.evaluation import IntermediateData + from trpc_agent_sdk.evaluation import Invocation + from trpc_agent_sdk.evaluation import RubricScore + from trpc_agent_sdk.types import Content + from trpc_agent_sdk.types import Part + + def _content(text: str, role: str) -> Content: + return Content(role=role, parts=[Part.from_text(text=text)]) + + expected = Invocation( + invocation_id="expected", + user_content=_content("只返回一行纯文本。", "user"), + final_response=_content("合格回答", "model"), + intermediate_data=IntermediateData(), + ) + actual = Invocation( + invocation_id="actual", + user_content=expected.user_content, + final_response=_content("第一行\n第二行", "model"), + intermediate_data=IntermediateData(), + ) + rubric_metric = EvalMetricResult( + metric_name="llm_rubric_response", + threshold=1.0, + score=0.5, + eval_status=EvalStatus.FAILED, + details=EvalMetricResultDetails( + reason="quality failed; format passed", + rubric_scores=[ + RubricScore(id="format", score=1.0, reason="single-line rule passed"), + RubricScore(id="quality", score=0.0, reason="required content missing"), + ], + ), + ) + case = EvalCase( + eval_id="explicit-format-pass", + eval_mode="trace", + conversation=[expected], + actual_conversation=[actual], + ) + run = EvalCaseResult( + eval_id=case.eval_id, + final_eval_status=EvalStatus.FAILED, + overall_eval_metric_results=[rubric_metric], + eval_metric_result_per_invocation=[ + EvalMetricResultPerInvocation( + actual_invocation=actual, + expected_invocation=expected, + eval_metric_results=[rubric_metric], + ) + ], + session_id="offline", + ) + + reasons = TraceEvaluator._attribute_failure(case, [run], {rubric_metric.metric_name: rubric_metric}) + + assert reasons[0].category == "llm_rubric_failure" + + +@pytest.mark.parametrize(("request_text", "expected", "actual"), [ + ("请用自然语言回答，不要返回 JSON。", '{"answer":"参考"}', "自然语言回答"), + ("请勿使用 JSON，请用自然语言回答。", '{"answer":"参考"}', "自然语言回答"), + ("切勿采用 JSON 格式。", '{"answer":"参考"}', "自然语言回答"), + ("不得返回 JSON。", '{"answer":"参考"}', "自然语言回答"), + ("不能使用 JSON。", '{"answer":"参考"}', "自然语言回答"), + ("不可输出 JSON。", '{"answer":"参考"}', "自然语言回答"), + ("Never return JSON.", '{"answer":"reference"}', "natural language answer"), + ("不要只返回一行，请分点说明。", "第一点\n第二点", "第一点\n第二点"), + ("不要使用 Markdown，请使用纯文本。", "# 参考标题", "纯文本回答"), +]) +def test_negative_format_directives_do_not_trigger_format_violations( + request_text: str, + expected: str, + actual: str, +) -> None: + _load_public_interface() + from eval_optimize_loop.loop.trace import TraceEvaluator + from trpc_agent_sdk.evaluation import EvalCase + from trpc_agent_sdk.evaluation import IntermediateData + from trpc_agent_sdk.evaluation import Invocation + from trpc_agent_sdk.types import Content + from trpc_agent_sdk.types import Part + + invocation = Invocation( + invocation_id="negative-format", + user_content=Content(role="user", parts=[Part.from_text(text=request_text)]), + final_response=Content(role="model", parts=[Part.from_text(text=expected)]), + intermediate_data=IntermediateData(), + ) + case = EvalCase(eval_id="negative-format", conversation=[invocation]) + + assert TraceEvaluator._format_violations(case, expected=expected, actual=actual) == [] + + +@pytest.mark.parametrize(("request_text", "expected", "actual", "violation"), [ + ("不要添加解释，只返回 JSON。", '{"ok":true}', "不是 JSON", "invalid_json"), + ("不要省略字段，请使用 JSON 返回。", '{"ok":true}', "不是 JSON", "invalid_json"), + ("不要赘述，只返回一行。", "合格回答", "第一行\n第二行", "not_single_line"), + ("不要赘述，请使用 Markdown 表格。", "| A |\n| --- |\n| B |", "纯文本", "missing_markdown_table"), + ("返回非空 JSON 对象。", '{"ok":true}', "不是 JSON", "invalid_json"), +]) +def test_content_negation_does_not_suppress_positive_format_directives( + request_text: str, + expected: str, + actual: str, + violation: str, +) -> None: + _load_public_interface() + from eval_optimize_loop.loop.trace import TraceEvaluator + from trpc_agent_sdk.evaluation import EvalCase + from trpc_agent_sdk.evaluation import IntermediateData + from trpc_agent_sdk.evaluation import Invocation + from trpc_agent_sdk.types import Content + from trpc_agent_sdk.types import Part + + invocation = Invocation( + invocation_id="positive-format", + user_content=Content(role="user", parts=[Part.from_text(text=request_text)]), + final_response=Content(role="model", parts=[Part.from_text(text=expected)]), + intermediate_data=IntermediateData(), + ) + case = EvalCase(eval_id="positive-format", conversation=[invocation]) + + assert violation in TraceEvaluator._format_violations(case, expected=expected, actual=actual) + + +def test_unevaluated_judge_is_attributed_as_evaluation_error() -> None: + _load_public_interface() + from eval_optimize_loop.loop.trace import TraceEvaluator + from trpc_agent_sdk.evaluation import EvalCase + from trpc_agent_sdk.evaluation import EvalCaseResult + from trpc_agent_sdk.evaluation import EvalMetricResult + from trpc_agent_sdk.evaluation import EvalMetricResultDetails + from trpc_agent_sdk.evaluation import EvalMetricResultPerInvocation + from trpc_agent_sdk.evaluation import EvalStatus + from trpc_agent_sdk.evaluation import IntermediateData + from trpc_agent_sdk.evaluation import Invocation + from trpc_agent_sdk.types import Content + from trpc_agent_sdk.types import Part + + expected = Invocation( + invocation_id="expected", + user_content=Content(role="user", parts=[Part.from_text(text="只返回 JSON。")]), + final_response=Content(role="model", parts=[Part.from_text(text='{"ok":true}')]), + intermediate_data=IntermediateData(), + ) + actual = Invocation( + invocation_id="actual", + user_content=expected.user_content, + final_response=Content(role="model", parts=[Part.from_text(text="judge unavailable")]), + intermediate_data=IntermediateData(), + ) + unavailable_metric = EvalMetricResult( + metric_name="llm_rubric_response", + threshold=1.0, + score=None, + eval_status=EvalStatus.NOT_EVALUATED, + details=EvalMetricResultDetails(reason="all judge models failed"), + ) + case = EvalCase( + eval_id="judge-unavailable", + eval_mode="trace", + conversation=[expected], + actual_conversation=[actual], + ) + run = EvalCaseResult( + eval_id=case.eval_id, + final_eval_status=EvalStatus.NOT_EVALUATED, + overall_eval_metric_results=[unavailable_metric], + eval_metric_result_per_invocation=[ + EvalMetricResultPerInvocation( + actual_invocation=actual, + expected_invocation=expected, + eval_metric_results=[unavailable_metric], + ) + ], + session_id="offline", + ) + + reasons = TraceEvaluator._attribute_failure(case, [run], {unavailable_metric.metric_name: unavailable_metric}) + + assert reasons[0].category == "evaluation_error" + assert reasons[0].evidence["reason"] == "all judge models failed" + assert not any(reason.category in {"format_failure", "llm_rubric_failure"} for reason in reasons) From 6a6a368e016b72f7cd72ab7e32494d1bc7abf0df Mon Sep 17 00:00:00 2001 From: Yun Hong <155888058+16yunH@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:20:43 +0800 Subject: [PATCH 2/2] examples: harden eval optimization regression loop Fail closed on ambiguous replay data, unsafe artifact labels, non-finite metrics, key-case pass regressions, and common tool error payloads. Isolate optimizer prompt writes from source files, reject unsafe in-process concurrency, and add adversarial regression coverage for gate, attribution, path, apply, and concurrency behavior. Fixes #91 RELEASE NOTES: Harden the evaluation and optimization example against ambiguous data, unsafe artifacts, and regression-gate edge cases. --- .../optimization/eval_optimize_loop/DESIGN.md | 2 +- .../optimization/eval_optimize_loop/README.md | 7 +- .../eval_optimize_loop/loop/analysis.py | 20 +- .../eval_optimize_loop/loop/models.py | 130 ++++++--- .../eval_optimize_loop/loop/offline.py | 6 + .../eval_optimize_loop/loop/pipeline.py | 140 +++++----- .../eval_optimize_loop/loop/trace.py | 39 ++- .../test_eval_optimize_loop_example.py | 247 ++++++++++++++++++ 8 files changed, 482 insertions(+), 109 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index 8e5c0ed6..6fbcbdde 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -1,3 +1,3 @@ # 设计说明（约 400 字） -闭环以 `EvalOptimizePipeline.run()` 作为唯一外部接口，把数据校验、基线评估、提示词优化、候选复评、差异计算、门禁和报告隐藏在同一深模块内。优化阶段直接复用 `AgentOptimizer`、`TargetPrompt` 与真实 GEPA；无密钥模式仅通过 `ModelRegistry` 注入离线 agent、judge 和 reflector，因而仍会经过 `LlmAgent`、内置 rubric 解析及优化器回调，不依赖 monkeypatch。外层将每个候选运行物化为 trace，再用 `AgentEvaluator` 独立评测 train 与 validation，避免采信优化器自己的聚合分数。候选集合按完整提示词哈希去重，即使优化器内部提前拒绝某次提案，外层仍会捕获并独立复评，防止遗漏验证回退。逐 case 差异区分新增通过、新增失败、升分、降分和不变；归因按工具执行、参数、格式、知识召回、rubric、最终回答的根因优先级输出证据。SDK 聚合器偶尔不保留逐 rubric 详情，格式归因依次采用详情、回放证据和请求中明确的 JSON、单行或 Markdown 约束，否定式要求不触发检查；judge 无有效 verdict 时单列 evaluation_error，避免把系统错误或内容失败误报为格式失败。Gate 对验证增益、配对 bootstrap 下界、新增 hard fail、关键 case、过拟合及资源预算做可配置 AND，并先过滤不安全候选，再计算质量、token、P95 时延 Pareto。运行前检查跨集合重复、近重复和答案泄漏；运行后保存完整 prompt、seed、哈希、耗时、成本、trace 与 JSON/Markdown 报告。默认始终恢复源 prompt，只有显式允许且 Gate 全通过才原子回写。 +闭环以 `EvalOptimizePipeline.run()` 作为唯一外部接口，把数据校验、基线评估、提示词优化、候选复评、差异计算、门禁和报告隐藏在同一深模块内。优化阶段直接复用 `AgentOptimizer`、`TargetPrompt` 与真实 GEPA；无密钥模式仅通过 `ModelRegistry` 注入离线 agent、judge 和 reflector，因而仍会经过 `LlmAgent`、内置 rubric 解析及优化器回调，不依赖 monkeypatch。外层将每个候选运行物化为 trace，再用 `AgentEvaluator` 独立评测 train 与 validation，避免采信优化器自己的聚合分数。候选集合按完整提示词哈希去重，即使优化器内部提前拒绝某次提案，外层仍会捕获并独立复评，防止遗漏验证回退。逐 case 差异区分新增通过、新增失败、升分、降分和不变；归因按工具执行、参数、格式、知识召回、rubric、最终回答的根因优先级输出证据。SDK 聚合器偶尔不保留逐 rubric 详情，格式归因依次采用详情、回放证据和请求中明确的 JSON、单行或 Markdown 约束，否定式要求不触发检查；judge 无有效 verdict 时单列 evaluation_error，避免把系统错误或内容失败误报为格式失败。Gate 对验证增益、配对 bootstrap 下界、新增 hard fail、关键 case、过拟合及资源预算做可配置 AND，并先过滤不安全候选，再计算质量、token、P95 时延 Pareto。运行前检查跨集合重复、近重复和答案泄漏；运行后保存完整 prompt、seed、哈希、耗时、成本、trace 与 JSON/Markdown 报告。默认仅在隔离 workspace 内改写 prompt，不触碰源文件，只有显式允许且 Gate 全通过才原子回写。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md index 213a0fa1..03d65ebb 100644 --- a/examples/optimization/eval_optimize_loop/README.md +++ b/examples/optimization/eval_optimize_loop/README.md @@ -18,7 +18,7 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py \ --output-dir /tmp/eval-optimize-loop ``` -命令正常完成时输出 `decision=accepted` 与报告路径。默认不会覆盖源 prompt；只有显式传入 `--apply-if-accepted` 且所有必选 Gate 均通过，才会原子回写最终候选。 +命令正常完成时输出 `decision=accepted` 与报告路径。优化器只改写输出目录中的隔离 prompt workspace，默认运行不会触碰源 prompt；只有显式传入 `--apply-if-accepted` 且所有必选 Gate 均通过，才会原子回写最终候选。 ## 为什么不是“伪造优化” @@ -71,7 +71,7 @@ reflection 2. validation pass-rate 增益达到 `min_validation_gain`； 3. paired bootstrap 区间下界达到配置值； 4. 不新增 hard failure； -5. key case 不降分； +5. key case 不允许从通过变失败，也不允许降分； 6. train 提升而 validation 下降时判定 overfit 并拒绝；趋势以 pass-rate 为主，持平时再比较 average score； 7. metric calls、token、耗时和成本均在预算内。 @@ -82,6 +82,7 @@ reflection - baseline 与 candidate 在同一 case 上配对，使用固定 seed 的 2,000 次 bootstrap，报告 validation pass-rate delta 的 95% 区间。 - 先过安全 Gate，再在合格候选中按 validation 质量、token 与 P95 latency 标记 Pareto 前沿。 - 运行前检查重复 id、train/validation 精确重复、去空白后的重复、相似度 ≥0.92 的近重复，以及 baseline/candidate prompt 中直接出现 validation reference answer；命中即 fail-closed。 +- artifact label 只允许有限长度的字母、数字、下划线和连字符，候选 id 必须唯一；离线回放遇到空问题或同一问题映射多个 case 时会拒绝运行，避免路径逃逸和静默错配。 - baseline、每个候选和每个 split 的 trace 均保存到 artifact 目录，可由 `AgentEvaluator` 重新播放。 ## 输出 @@ -112,4 +113,4 @@ pytest -q tests/evaluation/test_eval_optimize_loop_example.py ## 接入真实业务 -保留 `EvalOptimizePipeline.run()` 这个外部 interface，将 `offline.py` 的三个 adapter 替换为业务实现：`call_agent` 驱动真实 Agent，reflection/judge 的 `provider_name` 改为实际 provider，trace 物化器读取业务运行日志。`optimizer.json`、`TargetPrompt`、逐候选复评、diff、Gate 和报告模型无需改变。生产环境建议每次使用唯一 output directory；离线 registry 使用进程级固定 replay 配置，不支持在同一 Python 进程中并发启动两个 pipeline。 +保留 `EvalOptimizePipeline.run()` 这个外部 interface，将 `offline.py` 的三个 adapter 替换为业务实现：`call_agent` 驱动真实 Agent，reflection/judge 的 `provider_name` 改为实际 provider，trace 物化器读取业务运行日志。`optimizer.json`、`TargetPrompt`、逐候选复评、diff、Gate 和报告模型无需改变。生产环境应为每次运行使用唯一 output directory；不同进程会在各自 workspace 优化而不触碰源 prompt，同一 Python 进程中的并发离线运行会明确拒绝，避免共享 registry 串扰。 diff --git a/examples/optimization/eval_optimize_loop/loop/analysis.py b/examples/optimization/eval_optimize_loop/loop/analysis.py index 91217291..3c936979 100644 --- a/examples/optimization/eval_optimize_loop/loop/analysis.py +++ b/examples/optimization/eval_optimize_loop/loop/analysis.py @@ -83,6 +83,13 @@ def validate_data_quality( all_ids = [case.eval_id for case in train_set.eval_cases + validation_set.eval_cases] duplicate_ids = sorted(case_id for case_id, count in Counter(all_ids).items() if count > 1) + query_groups: dict[str, list[str]] = {} + for case in train_set.eval_cases + validation_set.eval_cases: + replay_query = self._case_replay_query(case) + if not replay_query: + raise ValueError(f"case {case.eval_id!r} has an empty user query") + query_groups.setdefault(replay_query, []).append(case.eval_id) + duplicate_queries = sorted(" == ".join(case_ids) for case_ids in query_groups.values() if len(case_ids) > 1) train_fingerprints = {self._case_fingerprint(case): case.eval_id for case in train_set.eval_cases} validation_fingerprints = {self._case_fingerprint(case): case.eval_id for case in validation_set.eval_cases} overlap = sorted(set(train_fingerprints) & set(validation_fingerprints)) @@ -106,9 +113,10 @@ def validate_data_quality( normalized = _normalize(expected) if len(normalized) >= 12 and normalized in prompt_normalized: leakage.append(case.eval_id) - if duplicate_ids or cross_split or near_cross_split or leakage: + if duplicate_ids or duplicate_queries or cross_split or near_cross_split or leakage: raise ValueError("data quality check failed: " - f"duplicate_ids={duplicate_ids}, cross_split={cross_split}, " + f"duplicate_ids={duplicate_ids}, duplicate_queries={duplicate_queries}, " + f"cross_split={cross_split}, " f"near_cross_split={near_cross_split}, prompt_leakage={leakage}") return DataQualityAudit( passed=True, @@ -131,6 +139,11 @@ def _case_normalized_content(case: EvalCase) -> str: for invocation in conversation) return _normalize(payload) + @staticmethod + def _case_replay_query(case: EvalCase) -> str: + conversation = case.conversation or [] + return _text(conversation[0].user_content) if conversation else "" + @staticmethod def _expected_response(case: EvalCase) -> str: conversation = case.conversation or [] @@ -324,7 +337,8 @@ def gate( baseline_validation_by_id = {case.case_id: case for case in baseline.validation.cases} key_regressions = sorted( case.case_id for case in candidate_validation.cases - if case.key_case and baseline_validation_by_id[case.case_id].score - case.score > epsilon) + if case.key_case and ((baseline_validation_by_id[case.case_id].passed and not case.passed) + or baseline_validation_by_id[case.case_id].score - case.score > epsilon)) def _trend(split_delta: SplitDelta) -> int: if split_delta.pass_rate_delta > epsilon: diff --git a/examples/optimization/eval_optimize_loop/loop/models.py b/examples/optimization/eval_optimize_loop/loop/models.py index ee3d1951..8093b6c0 100644 --- a/examples/optimization/eval_optimize_loop/loop/models.py +++ b/examples/optimization/eval_optimize_loop/loop/models.py @@ -10,6 +10,7 @@ from __future__ import annotations import json +import re from pathlib import Path from typing import Any from typing import Literal @@ -18,6 +19,39 @@ from pydantic import BaseModel from pydantic import ConfigDict from pydantic import Field +from pydantic import field_validator +from pydantic import model_validator + +_SAFE_ARTIFACT_LABEL = r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$" +_WINDOWS_DEVICE_NAMES = { + "aux", + "con", + "nul", + "prn", + *(f"com{index}" for index in range(1, 10)), + *(f"lpt{index}" for index in range(1, 10)), +} + + +def _validate_artifact_label( + value: str, + *, + subject: str, + reserved: set[str] | None = None, +) -> str: + if re.fullmatch(_SAFE_ARTIFACT_LABEL, value) is None: + raise ValueError(f"{subject} must be a safe artifact label") + folded = value.casefold() + if folded in _WINDOWS_DEVICE_NAMES or folded in (reserved or set()): + raise ValueError(f"{subject} is a reserved artifact label: {value!r}") + return value + + +def _casefold_duplicates(values: list[str]) -> list[str]: + groups: dict[str, list[str]] = {} + for value in values: + groups.setdefault(value.casefold(), []).append(value) + return sorted(" == ".join(group) for group in groups.values() if len(group) > 1) class _StrictModel(BaseModel): @@ -27,9 +61,18 @@ class _StrictModel(BaseModel): class CandidateSource(_StrictModel): """A deterministic prompt proposal used by the offline reflection model.""" - candidate_id: str + candidate_id: str = Field(pattern=_SAFE_ARTIFACT_LABEL) path: Path + @field_validator("candidate_id") + @classmethod + def _candidate_id_is_artifact_safe(cls, value: str) -> str: + return _validate_artifact_label( + value, + subject="candidate_id", + reserved={"baseline"}, + ) + class PipelineSpec(_StrictModel): """All paths and run controls needed by :class:`EvalOptimizePipeline`.""" @@ -40,14 +83,29 @@ class PipelineSpec(_StrictModel): gate_config: Path train_dataset: Path validation_dataset: Path - target_prompts: dict[str, Path] - candidate_sources: list[CandidateSource] + target_prompts: dict[str, Path] = Field(min_length=1) + candidate_sources: list[CandidateSource] = Field(min_length=1) output_dir: Path seed: int = 91 bootstrap_samples: int = Field(default=2000, ge=100) confidence_level: float = Field(default=0.95, gt=0.0, lt=1.0) apply_if_accepted: bool = False + @model_validator(mode="after") + def _validate_artifact_labels(self) -> "PipelineSpec": + candidate_ids = [source.candidate_id for source in self.candidate_sources] + duplicate_ids = _casefold_duplicates(candidate_ids) + if duplicate_ids: + raise ValueError(f"candidate_ids must be case-insensitively unique: {duplicate_ids}") + prompt_names = list(self.target_prompts) + duplicate_prompt_names = _casefold_duplicates(prompt_names) + if duplicate_prompt_names: + raise ValueError("target prompt names must be case-insensitively unique: " + f"{duplicate_prompt_names}") + for name in prompt_names: + _validate_artifact_label(name, subject="target prompt name") + return self + @classmethod def from_file( cls, @@ -94,8 +152,8 @@ def _resolve(value: str | Path) -> Path: class MetricOutcome(_StrictModel): metric_name: str - score: Optional[float] = None - threshold: float + score: Optional[float] = Field(default=None, allow_inf_nan=False) + threshold: float = Field(allow_inf_nan=False) passed: bool reason: str = "" @@ -115,7 +173,7 @@ class TrajectoryStep(_StrictModel): class CaseEvaluation(_StrictModel): case_id: str passed: bool - score: float + score: float = Field(allow_inf_nan=False) key_case: bool = False hard_fail: bool = False metrics: list[MetricOutcome] = Field(default_factory=list) @@ -128,8 +186,8 @@ class CaseEvaluation(_StrictModel): class SplitEvaluation(_StrictModel): split: Literal["train", "validation"] - pass_rate: float - average_score: float + pass_rate: float = Field(ge=0.0, le=1.0, allow_inf_nan=False) + average_score: float = Field(allow_inf_nan=False) cases: list[CaseEvaluation] @@ -152,24 +210,24 @@ class CaseDelta(_StrictModel): status: DeltaStatus baseline_passed: bool candidate_passed: bool - baseline_score: float - candidate_score: float - score_delta: float + baseline_score: float = Field(allow_inf_nan=False) + candidate_score: float = Field(allow_inf_nan=False) + score_delta: float = Field(allow_inf_nan=False) class PairedConfidenceInterval(_StrictModel): - point_estimate: float - lower: float - upper: float - confidence_level: float - bootstrap_samples: int + point_estimate: float = Field(allow_inf_nan=False) + lower: float = Field(allow_inf_nan=False) + upper: float = Field(allow_inf_nan=False) + confidence_level: float = Field(gt=0.0, lt=1.0, allow_inf_nan=False) + bootstrap_samples: int = Field(ge=1) seed: int class SplitDelta(_StrictModel): split: Literal["train", "validation"] - pass_rate_delta: float - average_score_delta: float + pass_rate_delta: float = Field(allow_inf_nan=False) + average_score_delta: float = Field(allow_inf_nan=False) paired_pass_rate_ci: PairedConfidenceInterval newly_passed: list[str] = Field(default_factory=list) newly_failed: list[str] = Field(default_factory=list) @@ -200,15 +258,15 @@ class GateDecision(_StrictModel): class ResourceUsage(_StrictModel): - metric_calls: int = 0 - reflection_calls: int = 0 - judge_calls: Optional[int] = None - prompt_tokens: int = 0 - completion_tokens: int = 0 - total_tokens: int = 0 - cost_usd: Optional[float] = None - duration_seconds: float = 0.0 - p95_latency_ms: Optional[float] = None + metric_calls: int = Field(default=0, ge=0) + reflection_calls: int = Field(default=0, ge=0) + judge_calls: Optional[int] = Field(default=None, ge=0) + prompt_tokens: int = Field(default=0, ge=0) + completion_tokens: int = Field(default=0, ge=0) + total_tokens: int = Field(default=0, ge=0) + cost_usd: Optional[float] = Field(default=None, ge=0.0, allow_inf_nan=False) + duration_seconds: float = Field(default=0.0, ge=0.0, allow_inf_nan=False) + p95_latency_ms: Optional[float] = Field(default=None, ge=0.0, allow_inf_nan=False) cost_measurement: str = "unavailable" @@ -236,17 +294,17 @@ class OptimizerAudit(_StrictModel): status: str stop_reason: Optional[str] = None used_agent_optimizer: bool - baseline_pass_rate: float - best_pass_rate: float - rounds: int + baseline_pass_rate: float = Field(ge=0.0, le=1.0, allow_inf_nan=False) + best_pass_rate: float = Field(ge=0.0, le=1.0, allow_inf_nan=False) + rounds: int = Field(ge=0) resources: ResourceUsage artifact_dir: str class DataQualityAudit(_StrictModel): passed: bool - train_cases: int - validation_cases: int + train_cases: int = Field(ge=0) + validation_cases: int = Field(ge=0) duplicate_ids: list[str] = Field(default_factory=list) cross_split_duplicates: list[str] = Field(default_factory=list) near_cross_split_duplicates: list[str] = Field(default_factory=list) @@ -257,7 +315,7 @@ class RunAudit(_StrictModel): run_id: str started_at: str finished_at: str - duration_seconds: float + duration_seconds: float = Field(ge=0.0, allow_inf_nan=False) seed: int config_sha256: str train_sha256: str @@ -267,9 +325,9 @@ class RunAudit(_StrictModel): class FailureAttributionSummary(_StrictModel): - explained_failed_cases: int - total_failed_cases: int - coverage_rate: float + explained_failed_cases: int = Field(ge=0) + total_failed_cases: int = Field(ge=0) + coverage_rate: float = Field(ge=0.0, le=1.0, allow_inf_nan=False) category_counts: dict[str, int] = Field(default_factory=dict) by_case: dict[str, list[FailureReason]] = Field(default_factory=dict) diff --git a/examples/optimization/eval_optimize_loop/loop/offline.py b/examples/optimization/eval_optimize_loop/loop/offline.py index 004e7069..542ee25b 100644 --- a/examples/optimization/eval_optimize_loop/loop/offline.py +++ b/examples/optimization/eval_optimize_loop/loop/offline.py @@ -93,6 +93,12 @@ def configure( if not conversation: continue query = _content_text(conversation[0].user_content) + if not query: + raise ValueError(f"case {case.eval_id!r} has an empty offline replay query") + if query in catalog: + previous = catalog[query] + raise ValueError("offline replay queries must be unique; " + f"cases {previous.eval_id!r} and {case.eval_id!r} share the same query") catalog[query] = case with cls._configuration_lock: cls._catalog = catalog diff --git a/examples/optimization/eval_optimize_loop/loop/pipeline.py b/examples/optimization/eval_optimize_loop/loop/pipeline.py index e609ad92..cffa69d4 100644 --- a/examples/optimization/eval_optimize_loop/loop/pipeline.py +++ b/examples/optimization/eval_optimize_loop/loop/pipeline.py @@ -11,6 +11,7 @@ import hashlib import json +import threading import time import uuid from datetime import datetime @@ -37,6 +38,8 @@ from .reporting import write_reports from .trace import TraceEvaluator +_OFFLINE_RUN_LOCK = threading.Lock() + def _sha256_bytes(payload: bytes) -> str: return hashlib.sha256(payload).hexdigest() @@ -75,6 +78,15 @@ def __init__(self, spec: PipelineSpec) -> None: self._spec = spec async def run(self) -> OptimizationReport: + """Execute one isolated offline run, rejecting unsafe concurrency.""" + if not _OFFLINE_RUN_LOCK.acquire(blocking=False): + raise RuntimeError("another offline evaluation/optimization pipeline is already running in this process") + try: + return await self._run_isolated() + finally: + _OFFLINE_RUN_LOCK.release() + + async def _run_isolated(self) -> OptimizationReport: """Run baseline, optimization, independent regression, gate, and reports.""" started_clock = time.perf_counter() started_at = datetime.now(timezone.utc) @@ -89,13 +101,12 @@ async def run(self) -> OptimizationReport: trace_evaluator = TraceEvaluator(spec.output_dir) train_set = EvalSet.model_validate_json(spec.train_dataset.read_text(encoding="utf-8")) validation_set = EvalSet.model_validate_json(spec.validation_dataset.read_text(encoding="utf-8")) + baseline_prompts = {name: path.read_text(encoding="utf-8") for name, path in spec.target_prompts.items()} known_candidates = { source.candidate_id: source.path.read_text(encoding="utf-8") for source in spec.candidate_sources } - prompt_text = "\n".join( - path.read_text(encoding="utf-8") - for path in (list(spec.target_prompts.values()) + [source.path for source in spec.candidate_sources])) + prompt_text = "\n".join([*baseline_prompts.values(), *known_candidates.values()]) data_quality = analyzer.validate_data_quality( train_set, validation_set, @@ -110,11 +121,19 @@ async def run(self) -> OptimizationReport: eval_sets=[train_set, validation_set], candidate_prompts=list(known_candidates.values()), ) + optimizer_dir = spec.output_dir / "optimizer" + prompt_workspace = optimizer_dir / "prompt_workspace" + prompt_workspace.mkdir(parents=True, exist_ok=True) target_prompt = TargetPrompt() + source_target_prompt = TargetPrompt() + working_prompt_paths: dict[str, Path] = {} for name, path in spec.target_prompts.items(): - target_prompt.add_path(name, str(path)) - baseline_prompts = await target_prompt.read_all() - call_agent = create_offline_call_agent(spec.target_prompts) + workspace_path = prompt_workspace / f"{name}.md" + workspace_path.write_text(baseline_prompts[name], encoding="utf-8") + working_prompt_paths[name] = workspace_path + target_prompt.add_path(name, str(workspace_path)) + source_target_prompt.add_path(name, str(path)) + call_agent = create_offline_call_agent(working_prompt_paths) baseline = BaselineEvaluation( train=await trace_evaluator.evaluate( @@ -133,7 +152,6 @@ async def run(self) -> OptimizationReport: ), ) - optimizer_dir = spec.output_dir / "optimizer" proposal_capture = _ProposalCapture() optimize_result = await AgentOptimizer.optimize( config_path=str(spec.optimizer_config), @@ -153,63 +171,59 @@ async def run(self) -> OptimizationReport: optimizer_resources = analyzer.optimizer_resources(optimize_result) candidates: list[CandidateEvaluation] = [] - try: - for optimizer_round, prompts in proposal_records: - candidate_started = time.perf_counter() - candidate_id = analyzer.candidate_id(prompts, known_candidates) - await target_prompt.write_all(prompts) - candidate_train = await trace_evaluator.evaluate( - train_set, - split="train", - eval_config=regression_config, - prompts=prompts, - trace_label=candidate_id, - ) - candidate_validation = await trace_evaluator.evaluate( - validation_set, - split="validation", - eval_config=regression_config, - prompts=prompts, - trace_label=candidate_id, - ) - delta = CandidateDelta( - train=analyzer.diff(baseline.train, candidate_train), - validation=analyzer.diff(baseline.validation, candidate_validation), - ) - candidate_resources = analyzer.candidate_resources( - train_set=train_set, - validation_set=validation_set, + for optimizer_round, prompts in proposal_records: + candidate_started = time.perf_counter() + candidate_id = analyzer.candidate_id(prompts, known_candidates) + candidate_train = await trace_evaluator.evaluate( + train_set, + split="train", + eval_config=regression_config, + prompts=prompts, + trace_label=candidate_id, + ) + candidate_validation = await trace_evaluator.evaluate( + validation_set, + split="validation", + eval_config=regression_config, + prompts=prompts, + trace_label=candidate_id, + ) + delta = CandidateDelta( + train=analyzer.diff(baseline.train, candidate_train), + validation=analyzer.diff(baseline.validation, candidate_validation), + ) + candidate_resources = analyzer.candidate_resources( + train_set=train_set, + validation_set=validation_set, + prompts=prompts, + eval_config=regression_config, + duration_seconds=time.perf_counter() - candidate_started, + ) + gate = analyzer.gate( + baseline=baseline, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + delta=delta, + optimizer_status=optimize_result.status, + resources=candidate_resources, + config=gate_config, + ) + candidates.append( + CandidateEvaluation( + candidate_id=candidate_id, prompts=prompts, - eval_config=regression_config, - duration_seconds=time.perf_counter() - candidate_started, - ) - gate = analyzer.gate( - baseline=baseline, - candidate_train=candidate_train, - candidate_validation=candidate_validation, + train=candidate_train, + validation=candidate_validation, delta=delta, - optimizer_status=optimize_result.status, - resources=candidate_resources, - config=gate_config, - ) - candidates.append( - CandidateEvaluation( - candidate_id=candidate_id, - prompts=prompts, - train=candidate_train, - validation=candidate_validation, - delta=delta, - gate=gate, - audit=CandidateAudit( - prompt_sha256=_sha256_text(json.dumps(prompts, sort_keys=True, ensure_ascii=False)), - source="GEPA on_proposal_end", - optimizer_round=optimizer_round, - seed=spec.seed, - resources=candidate_resources, - ), - )) - finally: - await target_prompt.write_all(baseline_prompts) + gate=gate, + audit=CandidateAudit( + prompt_sha256=_sha256_text(json.dumps(prompts, sort_keys=True, ensure_ascii=False)), + source="GEPA on_proposal_end", + optimizer_round=optimizer_round, + seed=spec.seed, + resources=candidate_resources, + ), + )) analyzer.mark_pareto(candidates) accepted_candidates = [candidate for candidate in candidates if candidate.gate.accepted] @@ -223,7 +237,7 @@ async def run(self) -> OptimizationReport: ), ) if accepted_candidates else None) if selected is not None and spec.apply_if_accepted: - await target_prompt.write_all(selected.prompts) + await source_target_prompt.write_all(selected.prompts) scoped_cases = { "baseline/train": baseline.train.cases, diff --git a/examples/optimization/eval_optimize_loop/loop/trace.py b/examples/optimization/eval_optimize_loop/loop/trace.py index d934dc31..e088c9f3 100644 --- a/examples/optimization/eval_optimize_loop/loop/trace.py +++ b/examples/optimization/eval_optimize_loop/loop/trace.py @@ -29,6 +29,9 @@ from .models import FailureReason from .models import MetricOutcome from .models import SplitEvaluation +from .models import _validate_artifact_label + +_SUPPORTED_SPLITS = {"train", "validation"} def _text(content: Any) -> str: @@ -42,6 +45,22 @@ def _payload_response_text(payload: dict[str, Any]) -> str: return "\n".join(str(part.get("text", "") or "") for part in response.get("parts", [])).strip() +def _tool_response_is_error(payload: Any) -> bool: + if not isinstance(payload, dict): + return False + status = str(payload.get("status", "")).casefold() + status_code = payload.get("status_code") + numeric_status_code = None + if isinstance(status_code, int) and not isinstance(status_code, bool): + numeric_status_code = status_code + elif isinstance(status_code, str) and status_code.strip().isdigit(): + numeric_status_code = int(status_code.strip()) + return bool( + payload.get("error") or status in {"error", "failed", "failure"} or payload.get("success") is False + or payload.get("ok") is False or payload.get("is_error") is True + or (numeric_status_code is not None and numeric_status_code >= 400)) + + class TraceEvaluator: """Replay deterministic traces through AgentEvaluator and explain failures.""" @@ -58,8 +77,10 @@ async def evaluate( trace_label: str, ) -> SplitEvaluation: """Materialize, persist, evaluate, and normalize one replay split.""" + if split not in _SUPPORTED_SPLITS: + raise ValueError(f"unsupported replay split: {split!r}") + trace_dir = self._trace_directory(trace_label) trace_set = self._materialize_trace(eval_set, prompts=prompts, trace_label=trace_label) - trace_dir = self._output_dir / "traces" / trace_label trace_dir.mkdir(parents=True, exist_ok=True) trace_path = trace_dir / f"{split}.trace.evalset.json" trace_path.write_text( @@ -84,6 +105,19 @@ async def evaluate( cases=cases, ) + def _trace_directory(self, trace_label: str) -> Path: + """Resolve a trace directory without allowing artifact-path escape.""" + _validate_artifact_label(trace_label, subject="trace label") + output_root = self._output_dir.resolve() + trace_root = (output_root / "traces").resolve() + trace_dir = (trace_root / trace_label).resolve() + try: + trace_root.relative_to(output_root) + trace_dir.relative_to(trace_root) + except ValueError as error: + raise ValueError("trace artifact path escapes the output directory") from error + return trace_dir + @staticmethod def _materialize_trace( eval_set: EvalSet, @@ -245,8 +279,7 @@ def _attribute_failure( error_responses = [] for response in actual_responses: payload = response.response - if isinstance(payload, dict) and (payload.get("error") - or str(payload.get("status", "")).lower() in {"error", "failed"}): + if _tool_response_is_error(payload): error_responses.append({ "tool": response.name, "id": response.id, diff --git a/tests/evaluation/test_eval_optimize_loop_example.py b/tests/evaluation/test_eval_optimize_loop_example.py index dadc45f0..7389b050 100644 --- a/tests/evaluation/test_eval_optimize_loop_example.py +++ b/tests/evaluation/test_eval_optimize_loop_example.py @@ -9,6 +9,7 @@ from __future__ import annotations +import asyncio import json import sys from pathlib import Path @@ -81,6 +82,198 @@ def test_output_override_is_resolved_from_callers_working_directory( assert spec.output_dir == tmp_path / "relative-run" +def test_artifact_labels_are_safe_and_candidate_ids_are_unique(tmp_path: Path, ) -> None: + _, PipelineSpec = _load_public_interface() + from eval_optimize_loop.loop.models import CandidateSource + from eval_optimize_loop.loop.trace import TraceEvaluator + + for unsafe_id in ("../outside", "baseline", "NUL"): + with pytest.raises(ValueError, match="candidate_id"): + CandidateSource(candidate_id=unsafe_id, path=tmp_path / "prompt.md") + with pytest.raises(ValueError, match="safe artifact label"): + TraceEvaluator(tmp_path)._trace_directory("../outside") + with pytest.raises(ValueError, match="reserved artifact label"): + TraceEvaluator(tmp_path)._trace_directory("CON") + + spec = PipelineSpec.from_file(_EXAMPLE_DIR / "pipeline.json", output_dir=tmp_path / "run") + payload = spec.model_dump() + payload["candidate_sources"][1]["candidate_id"] = payload["candidate_sources"][0]["candidate_id"].upper() + with pytest.raises(ValueError, match="candidate_ids must be case-insensitively unique"): + PipelineSpec.model_validate(payload) + + payload = spec.model_dump() + payload["target_prompts"]["SYSTEM"] = payload["target_prompts"]["system"] + with pytest.raises(ValueError, match="target prompt names must be case-insensitively unique"): + PipelineSpec.model_validate(payload) + + +def test_offline_model_rejects_ambiguous_duplicate_queries() -> None: + _load_public_interface() + from eval_optimize_loop.loop.offline import OfflineModel + from trpc_agent_sdk.evaluation import EvalSet + + train_payload = json.loads((_EXAMPLE_DIR / "data/train.evalset.json").read_text(encoding="utf-8")) + validation_payload = json.loads((_EXAMPLE_DIR / "data/val.evalset.json").read_text(encoding="utf-8")) + validation_payload["eval_cases"][0]["conversation"][0]["user_content"] = ( + train_payload["eval_cases"][0]["conversation"][0]["user_content"]) + train = EvalSet.model_validate(train_payload) + validation = EvalSet.model_validate(validation_payload) + + with pytest.raises(ValueError, match="offline replay queries must be unique"): + OfflineModel.configure( + eval_sets=[train, validation], + candidate_prompts=["[variant: test]"], + ) + + +def test_query_identity_preserves_case_and_inner_whitespace() -> None: + _load_public_interface() + from eval_optimize_loop.loop.analysis import RegressionAnalyzer + from eval_optimize_loop.loop.offline import OfflineModel + from trpc_agent_sdk.evaluation import EvalSet + + train_payload = json.loads((_EXAMPLE_DIR / "data/train.evalset.json").read_text(encoding="utf-8")) + validation_payload = json.loads((_EXAMPLE_DIR / "data/val.evalset.json").read_text(encoding="utf-8")) + train_payload["eval_cases"][0]["conversation"][0]["user_content"]["parts"][0]["text"] = "SKU ab c" + validation_payload["eval_cases"][0]["conversation"][0]["user_content"]["parts"][0]["text"] = "SKU a bc" + train = EvalSet.model_validate(train_payload) + validation = EvalSet.model_validate(validation_payload) + analyzer = RegressionAnalyzer(seed=91, bootstrap_samples=100, confidence_level=0.95) + + audit = analyzer.validate_data_quality( + train, + validation, + train_path=Path("train.evalset.json"), + validation_path=Path("val.evalset.json"), + prompt_text="", + ) + OfflineModel.configure( + eval_sets=[train, validation], + candidate_prompts=["[variant: test]"], + ) + + assert audit.passed is True + + +@pytest.mark.asyncio +async def test_cross_split_duplicate_queries_fail_data_quality_before_replay(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + validation_payload = json.loads((_EXAMPLE_DIR / "data/val.evalset.json").read_text(encoding="utf-8")) + train_payload = json.loads((_EXAMPLE_DIR / "data/train.evalset.json").read_text(encoding="utf-8")) + validation_payload["eval_cases"][0]["conversation"][0]["user_content"] = ( + train_payload["eval_cases"][0]["conversation"][0]["user_content"]) + validation_path = tmp_path / "duplicate-query.evalset.json" + validation_path.write_text(json.dumps(validation_payload, ensure_ascii=False), encoding="utf-8") + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "duplicate-query", + ).model_copy(update={"validation_dataset": validation_path}) + + with pytest.raises(ValueError, match="duplicate_queries"): + await EvalOptimizePipeline(spec).run() + + +def test_non_finite_scores_and_resources_fail_closed() -> None: + _load_public_interface() + from eval_optimize_loop.loop.models import CaseEvaluation + from eval_optimize_loop.loop.models import ResourceUsage + + with pytest.raises(ValueError, match="finite number"): + CaseEvaluation(case_id="nan-score", passed=False, score=float("nan")) + with pytest.raises(ValueError, match="finite number"): + ResourceUsage(duration_seconds=float("inf")) + + +@pytest.mark.parametrize("status_code", [500, "500"]) +def test_numeric_http_error_status_is_attributed_as_tool_error(status_code: int | str) -> None: + _load_public_interface() + from eval_optimize_loop.loop.trace import _tool_response_is_error + + assert _tool_response_is_error({"status_code": status_code, "message": "backend unavailable"}) is True + + +def test_key_case_pass_to_fail_is_rejected_even_when_score_is_unchanged() -> None: + _load_public_interface() + from eval_optimize_loop.loop.analysis import RegressionAnalyzer + from eval_optimize_loop.loop.models import BaselineEvaluation + from eval_optimize_loop.loop.models import CandidateDelta + from eval_optimize_loop.loop.models import CaseEvaluation + from eval_optimize_loop.loop.models import ResourceUsage + from eval_optimize_loop.loop.models import SplitEvaluation + + analyzer = RegressionAnalyzer(seed=91, bootstrap_samples=100, confidence_level=0.95) + baseline_train = SplitEvaluation( + split="train", + pass_rate=0.0, + average_score=0.0, + cases=[CaseEvaluation(case_id="train", passed=False, score=0.0)], + ) + candidate_train = baseline_train.model_copy(deep=True) + baseline_validation = SplitEvaluation( + split="validation", + pass_rate=0.5, + average_score=0.25, + cases=[ + CaseEvaluation(case_id="key", passed=True, score=0.5, key_case=True), + CaseEvaluation(case_id="other", passed=False, score=0.0), + ], + ) + candidate_validation = SplitEvaluation( + split="validation", + pass_rate=0.5, + average_score=0.75, + cases=[ + CaseEvaluation(case_id="key", passed=False, score=0.5, key_case=True), + CaseEvaluation(case_id="other", passed=True, score=1.0), + ], + ) + baseline = BaselineEvaluation(train=baseline_train, validation=baseline_validation) + delta = CandidateDelta( + train=analyzer.diff(baseline_train, candidate_train), + validation=analyzer.diff(baseline_validation, candidate_validation), + ) + + decision = analyzer.gate( + baseline=baseline, + candidate_train=candidate_train, + candidate_validation=candidate_validation, + delta=delta, + optimizer_status="SUCCEEDED", + resources=ResourceUsage(), + config={"min_validation_gain": 0.0}, + ) + + key_check = next(check for check in decision.checks if check.name == "key_cases_no_regression") + assert key_check.passed is False + assert key_check.actual == ["key"] + assert decision.accepted is False + + +@pytest.mark.asyncio +async def test_success_false_tool_response_is_attributed_as_tool_error(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + train_payload = json.loads((_EXAMPLE_DIR / "data/train.evalset.json").read_text(encoding="utf-8")) + tool_case = next(case for case in train_payload["eval_cases"] if case["eval_id"] == "train_tool_call_error") + traces = tool_case["session_input"]["state"]["variant_traces"] + for variant in ("baseline", "ineffective"): + traces[variant]["intermediate_data"]["tool_responses"][0]["response"] = { + "success": False, + "message": "backend unavailable", + } + train_path = tmp_path / "success-false.evalset.json" + train_path.write_text(json.dumps(train_payload, ensure_ascii=False), encoding="utf-8") + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "success-false", + ).model_copy(update={"train_dataset": train_path}) + + report = await EvalOptimizePipeline(spec).run() + + result = next(case for case in report.baseline.train.cases if case.case_id == "train_tool_call_error") + assert result.primary_failure == "tool_call_error" + assert result.failure_reasons[0].evidence["tool_responses"][0]["response"]["success"] is False + + @pytest.mark.asyncio async def test_offline_pipeline_writes_machine_and_human_reports_without_api_key( tmp_path: Path, @@ -111,6 +304,7 @@ async def test_offline_pipeline_writes_machine_and_human_reports_without_api_key assert report.optimizer.used_agent_optimizer is True assert report.selected_candidate_id is not None assert prompt_path.read_bytes() == original_prompt + assert (output_dir / "optimizer/prompt_workspace/system.md").is_file() assert (output_dir / "optimization_report.json").is_file() assert (output_dir / "optimization_report.md").is_file() @@ -123,6 +317,59 @@ async def test_offline_pipeline_writes_machine_and_human_reports_without_api_key assert "Gate" in markdown +@pytest.mark.asyncio +async def test_concurrent_offline_runs_fail_closed_without_touching_source_prompt(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + prompt_path = _EXAMPLE_DIR / "agent/prompts/system.md" + original_prompt = prompt_path.read_bytes() + specs = [ + PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / f"concurrent-{index}", + ) for index in range(2) + ] + + results = await asyncio.gather( + *(EvalOptimizePipeline(spec).run() for spec in specs), + return_exceptions=True, + ) + + reports = [result for result in results if not isinstance(result, BaseException)] + errors = [result for result in results if isinstance(result, BaseException)] + assert len(reports) == 1 + assert reports[0].status == "accepted" + assert len(errors) == 1 + assert isinstance(errors[0], RuntimeError) + assert "already running" in str(errors[0]) + assert prompt_path.read_bytes() == original_prompt + + +@pytest.mark.asyncio +async def test_explicit_apply_writes_only_the_accepted_prompt_atomically(tmp_path: Path, ) -> None: + EvalOptimizePipeline, PipelineSpec = _load_public_interface() + source_prompt = tmp_path / "system.md" + source_prompt.write_text( + (_EXAMPLE_DIR / "agent/prompts/system.md").read_text(encoding="utf-8"), + encoding="utf-8", + ) + spec = PipelineSpec.from_file( + _EXAMPLE_DIR / "pipeline.json", + output_dir=tmp_path / "apply", + ).model_copy(update={ + "target_prompts": { + "system": source_prompt + }, + "apply_if_accepted": True, + }) + + report = await EvalOptimizePipeline(spec).run() + + assert report.selected_candidate_id == "robust" + assert report.candidate is not None + assert source_prompt.read_text(encoding="utf-8") == report.candidate.prompts["system"] + assert not source_prompt.with_name(source_prompt.name + ".tmp").exists() + + @pytest.mark.asyncio async def test_every_unique_proposal_is_independently_re_evaluated_and_gated(tmp_path: Path, ) -> None: EvalOptimizePipeline, PipelineSpec = _load_public_interface()