diff --git a/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml b/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml index 9740aa4c1..1828b4d42 100644 --- a/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml +++ b/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml @@ -24,6 +24,29 @@ datasets: # Required benchmark default; set to true only for faster optimization/debug runs. stop_issuing_on_first_user_complete: false + - name: "aime25::gptoss" + type: "accuracy" + accuracy_config: + eval_method: "pass_at_1" + ground_truth: "answer" + extractor: "boxed_math_extractor" + num_repeats: 8 + + - name: "gpqa::gptoss" + type: "accuracy" + accuracy_config: + eval_method: "pass_at_1" + extractor: "abcd_extractor" + ground_truth: "ground_truth" + num_repeats: 5 + + - name: "livecodebench::gptoss" + type: "accuracy" + accuracy_config: + eval_method: "code_bench_scorer" + extractor: "python_code_extractor" + num_repeats: 3 + settings: runtime: min_duration_ms: 0 diff --git a/examples/10_Agentic_Inference/qwen_agentic_benchmark.yaml b/examples/10_Agentic_Inference/qwen_agentic_benchmark.yaml new file mode 100644 index 000000000..c9ca51abe --- /dev/null +++ b/examples/10_Agentic_Inference/qwen_agentic_benchmark.yaml @@ -0,0 +1,63 @@ +name: "qwen-agentic-benchmark" +version: "1.0" +type: "online" + +model_params: + name: "Qwen/Qwen3.6-35B-A3B" + temperature: 1.0 + top_k: 20 + top_p: 0.95 + repetition_penalty: 1.0 + presence_penalty: 1.5 + max_new_tokens: 8192 + chat_template_kwargs: + preserve_thinking: true + +datasets: + - name: agentic_coding + type: performance + path: /home/tianmuli/vllm_test/datasets/agentic_combined_v4.jsonl + accuracy_config: + eval_method: agentic_inference_inline # required benchmark default. + agentic_inference: + turn_timeout_s: 14400.0 + num_trajectories_to_issue: 1 + + - name: "aime25::gptoss" + type: "accuracy" + accuracy_config: + eval_method: "pass_at_1" + ground_truth: "answer" + extractor: "boxed_math_extractor" + num_repeats: 8 + + - name: "gpqa::gptoss" + type: "accuracy" + accuracy_config: + eval_method: "pass_at_1" + extractor: "abcd_extractor" + ground_truth: "ground_truth" + num_repeats: 5 + + - name: "livecodebench::gptoss" + type: "accuracy" + accuracy_config: + eval_method: "code_bench_scorer" + extractor: "python_code_extractor" + num_repeats: 3 + +settings: + runtime: + min_duration_ms: 0 + max_duration_ms: 36000000 + + load_pattern: + type: agentic_inference + target_concurrency: 128 # Submission-specific concurrency. + +endpoint_config: + endpoints: + - "http://localhost:30000" + api_type: openai + +report_dir: logs/qwen_agentic diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index a2050bbe3..8f63487d4 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -395,7 +395,20 @@ def setup_benchmark(config: BenchmarkConfig, test_mode: TestMode) -> BenchmarkCo dataloader, accuracy_datasets, eval_configs = _load_datasets(config, report_dir) # Setup runtime settings using factory method - rt_settings = RuntimeSettings.from_config(config, dataloader.num_samples()) + agentic_overrides: dict = {} + if isinstance(dataloader, AgenticInferenceDataset): + perf_cfgs = [d for d in config.datasets if d.type == DatasetType.PERFORMANCE] + agentic_cfg = perf_cfgs[0].agentic_inference if perf_cfgs else None + assert dataloader.conversation_metadata is not None + agentic_overrides = { + "agentic_num_conversations": dataloader.conversation_metadata.num_conversations, + "agentic_num_trajectories": agentic_cfg.num_trajectories_to_issue + if agentic_cfg is not None + else None, + } + rt_settings = RuntimeSettings.from_config( + config, dataloader.num_samples(), **agentic_overrides + ) # Calculate and display expected sample count total_samples = rt_settings.total_samples_to_issue() @@ -476,6 +489,7 @@ def _build_phases( # Accuracy phases — use eval_cfg.dataset_name as phase name so it matches # what Scorer._load_sample_index_map() looks up in sample_idx_map.json + perf_lp = ctx.rt_settings.load_pattern for eval_cfg in ctx.eval_configs: if eval_cfg.dataset_name == "performance": continue @@ -486,12 +500,17 @@ def _build_phases( "AgenticInferenceDataset, which is not yet supported for " "accuracy evaluation." ) - # Accuracy phases run at MAX_THROUGHPUT; inheriting perf_lp (e.g. POISSON) - # would silently rate-limit evaluation until an agentic inference accuracy strategy - # and QPS-budgeting support are added. - acc_load_pattern: LoadPattern | None = LoadPattern( - type=LoadPatternType.MAX_THROUGHPUT - ) + # Accuracy phases inherit the perf load pattern so the KV pool is not + # exhausted by simultaneous burst requests. AGENTIC_INFERENCE is + # downgraded to CONCURRENCY with the same cap because plain accuracy + # datasets are single-turn and cannot use the agentic scheduler. + if perf_lp is not None and perf_lp.type == LoadPatternType.AGENTIC_INFERENCE: + acc_load_pattern: LoadPattern | None = LoadPattern( + type=LoadPatternType.CONCURRENCY, + target_concurrency=perf_lp.target_concurrency, + ) + else: + acc_load_pattern = perf_lp acc_settings = RuntimeSettings( metric_target=ctx.rt_settings.metric_target, reported_metrics=ctx.rt_settings.reported_metrics, diff --git a/src/inference_endpoint/config/runtime_settings.py b/src/inference_endpoint/config/runtime_settings.py index 5067c78a1..449f12f0b 100644 --- a/src/inference_endpoint/config/runtime_settings.py +++ b/src/inference_endpoint/config/runtime_settings.py @@ -85,6 +85,12 @@ class RuntimeSettings: load_pattern: LoadPattern | None """Load pattern configuration""" + agentic_num_trajectories: int | None = None + """For agentic inference: num_trajectories_to_issue from dataset config (None = all).""" + + agentic_num_conversations: int | None = None + """For agentic inference: total distinct conversations in the loaded dataset.""" + @classmethod def from_config( cls, @@ -200,19 +206,37 @@ def total_samples_to_issue( self.load_pattern is not None and self.load_pattern.type == LoadPatternType.AGENTIC_INFERENCE ): - if self.n_samples_from_dataset < self.min_sample_count: + total = self.n_samples_from_dataset + if ( + self.agentic_num_trajectories is not None + and self.agentic_num_conversations is not None + and self.agentic_num_conversations > 0 + ): + # Scale proportionally: turns_per_trajectory ≈ total_turns / num_conversations + total = max( + 1, + round( + self.n_samples_from_dataset + * self.agentic_num_trajectories + / self.agentic_num_conversations + ), + ) + if total < self.min_sample_count: logger.warning( "Agentic inference run: min_sample_count=%d exceeds dataset " "client-turn count=%d; using dataset size. Agentic inference cannot " "issue more samples than the dataset provides.", self.min_sample_count, - self.n_samples_from_dataset, + total, ) logger.debug( - "Sample count: %d (agentic inference: issuing all client turns)", - self.n_samples_from_dataset, + "Sample count: %d (agentic inference: %s)", + total, + f"{self.agentic_num_trajectories} trajectories × avg turns" + if self.agentic_num_trajectories is not None + else "issuing all client turns", ) - return self.n_samples_from_dataset + return total # If min_duration is 0, use all dataset samples (new CLI default behavior) if self.min_duration_ms == 0: diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index 9226d7f85..a7ff1ed29 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -273,14 +273,14 @@ class AgenticInferenceConfig(BaseModel): ), ) enable_salt: bool = Field( - False, + True, description=( "Add deterministic salt markers before and after the system prompt " "to prevent KV cache reuse across trajectories in agentic inference setting." ), ) inject_tool_delay: bool = Field( - False, + True, description=( "Pause for a predefined duration between turns. Duration is defined " "in dataset." diff --git a/tests/integration/test_agentic_inference.py b/tests/integration/test_agentic_inference.py index 8cc4142ff..c8081685b 100644 --- a/tests/integration/test_agentic_inference.py +++ b/tests/integration/test_agentic_inference.py @@ -93,6 +93,7 @@ def _make_strategy( inject_tool_delay: bool = False, ) -> AgenticInferenceStrategy: agentic_cfg = AgenticInferenceConfig( + enable_salt=False, turn_timeout_s=10.0, inject_tool_delay=inject_tool_delay, ) @@ -377,7 +378,7 @@ async def test_turn_ordering_enforced_end_to_end(echo_server): {"conversation_id": "c1", "turn": 3, "role": "user", "content": "Second"}, ] ds = _make_dataset(rows) - agentic_cfg = AgenticInferenceConfig(turn_timeout_s=10.0) + agentic_cfg = AgenticInferenceConfig(enable_salt=False, turn_timeout_s=10.0) conv_manager = ConversationManager() strategy = AgenticInferenceStrategy( conversation_manager=conv_manager, diff --git a/tests/unit/load_generator/test_agentic_inference_strategy.py b/tests/unit/load_generator/test_agentic_inference_strategy.py index b183b876f..7c336865d 100644 --- a/tests/unit/load_generator/test_agentic_inference_strategy.py +++ b/tests/unit/load_generator/test_agentic_inference_strategy.py @@ -158,7 +158,7 @@ def _make_dataset_metadata(conversations: dict[str, list[int]]) -> ConversationM async def test_first_user_complete_stops_tracking_but_can_continue_for_accuracy(): conv_manager = ConversationManager() metadata = _make_dataset_metadata({"conv1": [1], "conv2": [1, 2]}) - cfg = AgenticInferenceConfig(num_trajectories_to_issue=2) + cfg = AgenticInferenceConfig(enable_salt=False, num_trajectories_to_issue=2) strategy = AgenticInferenceStrategy( conv_manager, metadata, @@ -209,6 +209,7 @@ async def test_stop_on_first_user_complete_refills_until_budget_exhausted(): conv_manager = ConversationManager() metadata = _make_dataset_metadata({"conv1": [1], "conv2": [1], "conv3": [1]}) cfg = AgenticInferenceConfig( + enable_salt=False, stop_issuing_on_first_user_complete=True, num_trajectories_to_issue=3, ) @@ -766,7 +767,9 @@ async def test_abort_remaining_turns_includes_pending_delayed_turn(): conv_manager = ConversationManager() conv_manager.get_or_create("c1", expected_client_turns=3) metadata = _metadata_with_delay("c1", [1, 2, 3], delay_turn=2, delay=60.0) - cfg = AgenticInferenceConfig(turn_timeout_s=5.0, inject_tool_delay=True) + cfg = AgenticInferenceConfig( + enable_salt=False, turn_timeout_s=5.0, inject_tool_delay=True + ) strategy = AgenticInferenceStrategy( conv_manager, metadata, agentic_inference_config=cfg ) @@ -814,7 +817,9 @@ async def test_execute_waits_for_delayed_first_turns(): conv_manager = ConversationManager() metadata = _make_dataset_metadata({"c1": [1], "c2": [1]}) metadata.delay_seconds_by_key = {("c1", 1): 0.02, ("c2", 1): 0.02} - cfg = AgenticInferenceConfig(turn_timeout_s=5.0, inject_tool_delay=True) + cfg = AgenticInferenceConfig( + enable_salt=False, turn_timeout_s=5.0, inject_tool_delay=True + ) strategy = AgenticInferenceStrategy( conv_manager, metadata, @@ -851,7 +856,9 @@ async def test_inject_tool_delay_defers_issue_via_call_later(): conv_manager = ConversationManager() metadata = _metadata_with_delay("c1", [1, 2], delay_turn=2, delay=0.05) - cfg = AgenticInferenceConfig(turn_timeout_s=5.0, inject_tool_delay=True) + cfg = AgenticInferenceConfig( + enable_salt=False, turn_timeout_s=5.0, inject_tool_delay=True + ) strategy = AgenticInferenceStrategy( conv_manager, metadata, agentic_inference_config=cfg ) @@ -998,7 +1005,9 @@ async def test_inject_tool_delay_disabled_issues_immediately(): conv_manager = ConversationManager() metadata = _metadata_with_delay("c1", [1, 2], delay_turn=2, delay=2.0) - cfg = AgenticInferenceConfig(turn_timeout_s=5.0, inject_tool_delay=False) + cfg = AgenticInferenceConfig( + enable_salt=False, turn_timeout_s=5.0, inject_tool_delay=False + ) strategy = AgenticInferenceStrategy( conv_manager, metadata, agentic_inference_config=cfg ) @@ -1049,7 +1058,9 @@ async def test_inject_tool_delay_no_dataset_field_back_compat(): conv_manager = ConversationManager() metadata = _make_dataset_metadata({"c1": [1, 2]}) - cfg = AgenticInferenceConfig(turn_timeout_s=5.0, inject_tool_delay=True) + cfg = AgenticInferenceConfig( + enable_salt=False, turn_timeout_s=5.0, inject_tool_delay=True + ) strategy = AgenticInferenceStrategy( conv_manager, metadata, agentic_inference_config=cfg ) @@ -1082,7 +1093,9 @@ async def test_inject_tool_delay_cancels_on_timeout(): conv_manager = ConversationManager() metadata = _metadata_with_delay("c1", [1, 2, 3], delay_turn=3, delay=1.0) - cfg = AgenticInferenceConfig(turn_timeout_s=0.1, inject_tool_delay=True) + cfg = AgenticInferenceConfig( + enable_salt=False, turn_timeout_s=0.1, inject_tool_delay=True + ) strategy = AgenticInferenceStrategy( conv_manager, metadata, agentic_inference_config=cfg )