From 5b2c348155f1dec8e138c663254ba45338f2a474 Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Mon, 13 Jul 2026 13:19:18 -0700 Subject: [PATCH 1/5] feat(config): consolidate all durations and deadlines into one Timeouts model One frozen pydantic model (config/timeouts.py, mounted at settings.timeouts) now owns every global time knob: min/max duration, per-phase drain deadlines, service-ready deadline, metrics drain budget, and the whole-run watchdog. - BenchmarkConfig.timeout was a silent no-op consumed nowhere; it is now settings.timeouts.run_timeout_s (--timeout alias preserved): a real whole-run watchdog that SIGTERMs the metrics aggregator (INTERRUPTED final snapshot), stops the session, and exits non-zero. A fired watchdog can never produce a COMPLETE report (locked by integration test). - DrainConfig deleted; drain fields moved to settings.timeouts with CLI aliases unchanged. metrics_drain_timeout_s normalized to None=unlimited (was 0=unlimited); the aggregator argv boundary still speaks 0. - metrics_tokenizer_workers is not a timeout: moved to settings.metrics (new MetricsConfig block). - min/max_duration_ms (+ suffix parsing + cross-validator) moved from RuntimeConfig to Timeouts; RuntimeSettings resolves them to plain values at setup, so nothing in the hot path reads pydantic. - ServiceLauncher.terminate_all() added (graceful SIGTERM counterpart to kill_all) for the watchdog path. - Templates regenerated; examples/docs/tests migrated (YAML keys moved, no back-compat shims per repo convention). Example run_timeout_s values were dropped rather than carried over: the old top-level timeout was inert, and enforcing stale values would abort valid runs. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 7 +- docs/CLI_DESIGN.md | 2 +- docs/CLI_QUICK_REFERENCE.md | 4 +- docs/config/DESIGN.md | 4 +- .../offline_llama3_8b_cnn.yaml | 6 +- .../online_llama2_70b_cnn.yaml | 6 +- .../compare_with_vllm.py | 5 +- .../gptoss_120b_example.yaml | 6 +- .../sglang_gptoss_120b_example.yaml | 7 +- .../vllm_gptoss_120b_example.yaml | 7 +- ...m_gptoss_120b_per_dataset_osl_example.yaml | 7 +- .../offline_llama3_8b_cnn.yaml | 6 +- .../online_llama3_8b_cnn.yaml | 6 +- .../online_llama2_70b_orca.yaml | 6 +- ...ractive_qwen3_vl_235b_a22b_shopify_8k.yaml | 5 +- .../offline_qwen3_vl_235b_a22b_shopify.yaml | 14 +- .../server_qwen3_vl_235b_a22b_shopify.yaml | 7 +- .../offline_wan22.yaml | 4 +- .../offline_wan22_accuracy.yaml | 4 +- .../offline_wan22_submission.yaml | 8 +- .../single_stream_wan22_submission.yaml | 8 +- examples/10_Agentic_Inference/README.md | 4 +- .../kimi_agentic_benchmark.yaml | 2 +- .../online_edge_full_run.yaml | 8 +- scripts/regenerate_templates.py | 4 +- .../async_utils/services/launcher.py | 11 ++ .../commands/benchmark/cli.py | 10 +- .../commands/benchmark/execute.py | 69 ++++++- .../config/runtime_settings.py | 7 +- src/inference_endpoint/config/schema.py | 120 +----------- .../templates/concurrency_template.yaml | 5 +- .../templates/concurrency_template_full.yaml | 23 +-- .../config/templates/offline_template.yaml | 5 +- .../templates/offline_template_full.yaml | 23 +-- .../config/templates/online_template.yaml | 5 +- .../templates/online_template_full.yaml | 23 +-- .../config/templates/submission_template.yaml | 6 +- src/inference_endpoint/config/timeouts.py | 175 ++++++++++++++++++ .../commands/test_accuracy_pipeline.py | 4 +- .../commands/test_benchmark_command.py | 14 +- .../integration/commands/test_run_timeout.py | 79 ++++++++ tests/integration/commands/test_warmup.py | 4 +- tests/unit/commands/test_benchmark.py | 85 ++++----- tests/unit/config/test_schema.py | 10 +- tests/unit/config/test_yaml_loader.py | 6 +- 45 files changed, 546 insertions(+), 285 deletions(-) create mode 100644 src/inference_endpoint/config/timeouts.py create mode 100644 tests/integration/commands/test_run_timeout.py diff --git a/AGENTS.md b/AGENTS.md index 81f222eb9..1b51edb62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,7 +92,7 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint | **Dataset Manager** | `src/inference_endpoint/dataset_manager/` | Loads JSONL, HuggingFace, CSV, JSON, Parquet datasets. `Dataset` base class with `load_sample()`/`num_samples()` interface | | **Metrics Aggregator** | `src/inference_endpoint/async_utils/services/metrics_aggregator/` | Subprocess. Subscribes to events, aggregates per-sample metrics into a `MetricsRegistry` (counters + HDR-histogram series + raw values), publishes `MetricsSnapshot` over IPC PUB at a configurable cadence (`SessionState`: `INITIALIZE` → `LIVE` → `DRAINING` → {`COMPLETE` \| `INTERRUPTED`}). Final snapshot is atomically written to `final_snapshot.json` as the **primary** Report source; the terminal pub/sub frame is a TUI "run finished" signal only. | | **Report** | `src/inference_endpoint/metrics/report.py` | `Report.from_snapshot(dict)` — pure-function builder consuming the dict form (`snapshot_to_dict`). Reads `final_snapshot.json` directly via `json.loads` (no Struct decode). Plumbs `complete = (state == "complete" and n_pending_tasks == 0)`; renders an explicit warning for `INTERRUPTED` runs. | -| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema (`schema.py`), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | +| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema (`schema.py`), `Timeouts` (`config/timeouts.py` — all global durations/deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | | **CLI** | `src/inference_endpoint/main.py`, `commands/benchmark/cli.py` | cyclopts-based, auto-generated from `schema.py` and `HTTPClientConfig` Pydantic models. Flat shorthands via `cyclopts.Parameter(alias=...)` | | **Async Utils** | `src/inference_endpoint/async_utils/` | `LoopManager` (uvloop + eager_task_factory), ZMQ transport layer, generic `MessageCodec[T]`-parametrized pub/sub, event publisher | | **OpenAI/SGLang** | `src/inference_endpoint/openai/`, `sglang/` | Protocol adapters and response accumulators for different API formats. `openai_completions` adapter (`completions_adapter.py`) sends pre-tokenized token IDs to `/v1/completions`, bypassing the server chat template — required for gpt-oss-120b on vLLM. `sglang` adapter sends to `/generate` via `input_ids`. Both apply `Harmonize()` client-side. | @@ -118,7 +118,7 @@ The aggregator is a separate process (`python -m inference_endpoint.async_utils. - **Series storage**: each `SeriesSampler` keeps three parallel views: O(1) cheap rollups (count/total/min/max/sum_sq, exact), an HDR Histogram (cheap live percentiles), and an in-memory `array.array` of raw values (for exact percentiles in the `COMPLETE` snapshot). Hot path is `registry.record(name, value)` — no allocation, no I/O. - **Counter API**: `registry.increment(name, delta=1)` for sample-event counters. `registry.set_counter(name, value)` only for the three derived-duration counters (`total_duration_ns` max-of-elapsed, `tracked_duration_ns` sum-of-blocks, `legacy_loadgen_window_duration_ns` first-issue→last-issued-completion span for LoadGen-parity QPS/TPS). -- **Lifecycle**: `INITIALIZE` (constructed, awaiting first `STARTED`) → `LIVE` (run in progress, ticking every `--publish-interval` seconds) → `DRAINING` (set on `ENDED`; tick continues; bounded by the `--drain-timeout` budget — schema default 0 = unlimited) → terminal: `COMPLETE` (clean end via `publish_final`, exact stats) **or** `INTERRUPTED` (signal-handler-triggered final via SIGTERM/SIGINT; best-effort partial stats). Drain timeout detected by consumers as `state == COMPLETE and n_pending_tasks > 0`; interrupted runs are detected as `state == INTERRUPTED` directly. +- **Lifecycle**: `INITIALIZE` (constructed, awaiting first `STARTED`) → `LIVE` (run in progress, ticking every `--publish-interval` seconds) → `DRAINING` (set on `ENDED`; tick continues; bounded by the `--drain-timeout` budget — argv 0 = unlimited; schema `settings.timeouts.metrics_drain_timeout_s` uses None = unlimited, converted at the argv boundary) → terminal: `COMPLETE` (clean end via `publish_final`, exact stats) **or** `INTERRUPTED` (signal-handler-triggered final via SIGTERM/SIGINT; best-effort partial stats). Drain timeout detected by consumers as `state == COMPLETE and n_pending_tasks > 0`; interrupted runs are detected as `state == INTERRUPTED` directly. - **Final delivery is dual-path with separated concerns**: `publish_final` atomically writes `final_snapshot.json` (`tmp + fsync(file) + rename + fsync(parent_dir)`) — this is the **primary** Report source — AND emits the terminal-state snapshot over pub/sub as a TUI shutdown signal. Each path is wrapped in its own try/except so one failure cannot suppress the other. Main process consumer reads `final_snapshot.json` (via `json.loads` to dict, no Struct decode); falls back to the subscriber's `latest` live snapshot only if the file is missing (e.g. SIGKILL / OOM before the signal handler ran). The dict form is the canonical consumer contract (see `snapshot_to_dict`). - **Histogram bucket edges are dynamic per snapshot**: log-spaced over the observed `[min, max]`. Bucket count is fixed at construction; consumers MUST re-render from the snapshot's `(lo, hi, count)` triples each frame and MUST NOT track bucket-by-index across snapshots. @@ -127,7 +127,7 @@ The aggregator is a separate process (`python -m inference_endpoint.async_utils. CLI is auto-generated from `config/schema.py` Pydantic models via cyclopts. Fields annotated with `cyclopts.Parameter(alias="--flag")` get flat shorthands; all other fields get auto-generated dotted flags (kebab-case). - **CLI mode** (`offline`/`online`): cyclopts constructs `OfflineBenchmarkConfig`/`OnlineBenchmarkConfig` (subclasses in `config/schema.py`) directly from CLI args. Type locked via `Literal`. `--dataset` is repeatable with TOML-style format `[perf|acc:][,key=value...]` (e.g. `--dataset data.csv,samples=500,parser.prompt=article`). Full accuracy support via `accuracy_config.eval_method=pass_at_1` etc. -- **YAML mode** (`from-config`): `BenchmarkConfig.from_yaml_file()` loads YAML, resolves env vars, and auto-selects the right subclass via Pydantic discriminated union. Optional `--timeout`/`--mode` overrides via `config.with_updates()`. +- **YAML mode** (`from-config`): `BenchmarkConfig.from_yaml_file()` loads YAML, resolves env vars, and auto-selects the right subclass via Pydantic discriminated union. Optional `--timeout` (maps to `settings.timeouts.run_timeout_s`)/`--mode` overrides via `config.with_updates()`. - **eval**: Not yet implemented (raises `CLIError` with a tracking issue link) ### Config Construction & Validation @@ -240,6 +240,7 @@ src/inference_endpoint/ │ └── results_plots.py # Standardized run-artifact plots (matplotlib-guarded); CLI: scripts/plot_results.py ├── config/ │ ├── schema.py # Single source of truth: Pydantic models + cyclopts annotations +│ ├── timeouts.py # Timeouts — all global durations + deadlines (settings.timeouts) │ ├── runtime_settings.py # RuntimeSettings + SampleOrderSpec dataclasses │ ├── ruleset_base.py # BenchmarkSuiteRuleset base │ ├── ruleset_registry.py # Ruleset registry diff --git a/docs/CLI_DESIGN.md b/docs/CLI_DESIGN.md index da4799a14..bbdc76b1a 100644 --- a/docs/CLI_DESIGN.md +++ b/docs/CLI_DESIGN.md @@ -204,5 +204,5 @@ class HTTPClientConfig(WithUpdatesMixin, BaseModel): `BenchmarkConfig` is frozen. Use `with_updates()` to produce new instances with re-validation: ```python -config = config.with_updates(timeout=300, datasets=["new_data.jsonl"]) +config = config.with_updates(report_dir="results/run1", datasets=["new_data.jsonl"]) ``` diff --git a/docs/CLI_QUICK_REFERENCE.md b/docs/CLI_QUICK_REFERENCE.md index e16593463..5a2c545b8 100644 --- a/docs/CLI_QUICK_REFERENCE.md +++ b/docs/CLI_QUICK_REFERENCE.md @@ -96,7 +96,7 @@ Flag names shown as `--full.dotted.path --alias`. Both forms work. - `--model-params.max-new-tokens --max-output-tokens` - Max output tokens (default: 1024) - `--model-params.osl-distribution.min --min-output-tokens` - Min output tokens (default: 1) - `--model-params.streaming --streaming` - Streaming mode: auto/on/off (default: auto) -- `--runtime.min-duration-ms --duration` - Min duration: ms default, or with suffix (600s, 10m) (default: 600000) +- `--min-duration-ms --duration` - Min duration: ms default, or with suffix (600s, 10m) (default: 600000) - `--runtime.n-samples-to-issue --num-samples` - Explicit sample count override - `--client.num-workers --workers` - HTTP workers (-1=auto, default: -1) - `--client.max-connections --max-connections` - Max TCP connections (-1=unlimited) @@ -106,7 +106,7 @@ Flag names shown as `--full.dotted.path --alias`. Both forms work. Note: applies to CLI-driven `benchmark offline` / `benchmark online`; `benchmark from-config` does not expose a CLI override for `report_dir`. Set it in the YAML only if you need to control the output location; otherwise a default report directory is used. -- `--timeout` - Global timeout in seconds +- `--timeout` - Whole-run watchdog in seconds (off by default). If it fires, the run is aborted and the report is marked INTERRUPTED (exits non-zero). - `--enable-cpu-affinity / --no-cpu-affinity` - NUMA-aware CPU pinning (default: true) **Online-specific:** diff --git a/docs/config/DESIGN.md b/docs/config/DESIGN.md index 795efbb66..5064157c1 100644 --- a/docs/config/DESIGN.md +++ b/docs/config/DESIGN.md @@ -61,8 +61,8 @@ Immutable snapshot of all parameters needed to execute a run. | -------------------- | -------------- | --------------------------------------- | | `load_pattern` | `LoadPattern` | config | | `n_samples_to_issue` | `int` | calculated: QPS × duration, or explicit | -| `min_duration_ms` | `int` | runtime config | -| `max_duration_ms` | `int` | runtime config | +| `min_duration_ms` | `int` | `settings.timeouts` | +| `max_duration_ms` | `int` | `settings.timeouts` | | `min_sample_count` | `int` | current default / future ruleset hook | | `metric_target` | `Metric` | primary target driving scheduler logic | | `reported_metrics` | `list[Metric]` | metrics validated after the run | diff --git a/examples/02_ServerBenchmarking/offline_llama3_8b_cnn.yaml b/examples/02_ServerBenchmarking/offline_llama3_8b_cnn.yaml index bc5b92f1d..17ce0d593 100644 --- a/examples/02_ServerBenchmarking/offline_llama3_8b_cnn.yaml +++ b/examples/02_ServerBenchmarking/offline_llama3_8b_cnn.yaml @@ -19,11 +19,13 @@ datasets: settings: runtime: - min_duration_ms: 6000 # 6 seconds - max_duration_ms: 60000 # 1 minute scheduler_random_seed: 137 # For Poisson/distribution sampling dataloader_random_seed: 111 # For dataset shuffling + timeouts: + min_duration_ms: 6000 # 6 seconds + max_duration_ms: 60000 # 1 minute + load_pattern: type: "max_throughput" diff --git a/examples/02_ServerBenchmarking/online_llama2_70b_cnn.yaml b/examples/02_ServerBenchmarking/online_llama2_70b_cnn.yaml index d16035447..497d47579 100644 --- a/examples/02_ServerBenchmarking/online_llama2_70b_cnn.yaml +++ b/examples/02_ServerBenchmarking/online_llama2_70b_cnn.yaml @@ -19,11 +19,13 @@ datasets: settings: runtime: - min_duration_ms: 60000 # 1 minute - max_duration_ms: 180000 # 3 minutes scheduler_random_seed: 42 # For Poisson/distribution sampling dataloader_random_seed: 42 # For dataset shuffling + timeouts: + min_duration_ms: 60000 # 1 minute + max_duration_ms: 180000 # 3 minutes + load_pattern: type: "poisson" target_qps: 10 diff --git a/examples/03_BenchmarkComparison/compare_with_vllm.py b/examples/03_BenchmarkComparison/compare_with_vllm.py index d28638938..1f8263f23 100644 --- a/examples/03_BenchmarkComparison/compare_with_vllm.py +++ b/examples/03_BenchmarkComparison/compare_with_vllm.py @@ -177,16 +177,17 @@ def generate_ie_config( ], "settings": { "runtime": { + "n_samples_to_issue": num_requests, + }, + "timeouts": { "min_duration_ms": 0, "max_duration_ms": timeout * 1000, - "n_samples_to_issue": num_requests, }, "load_pattern": {"type": "max_throughput"}, "client": {"num_workers": workers}, }, "endpoint_config": {"endpoints": [endpoint_url]}, "report_dir": str(report_dir), - "timeout": timeout, } with open(config_path, "w") as f: diff --git a/examples/04_GPTOSS120B_Example/gptoss_120b_example.yaml b/examples/04_GPTOSS120B_Example/gptoss_120b_example.yaml index 0bd3a2571..14a96e04b 100644 --- a/examples/04_GPTOSS120B_Example/gptoss_120b_example.yaml +++ b/examples/04_GPTOSS120B_Example/gptoss_120b_example.yaml @@ -17,11 +17,13 @@ datasets: settings: runtime: - min_duration_ms: 300 - max_duration_ms: 6000 scheduler_random_seed: 42 dataloader_random_seed: 42 + timeouts: + min_duration_ms: 300 + max_duration_ms: 6000 + load_pattern: type: "concurrency" target_concurrency: 512 diff --git a/examples/04_GPTOSS120B_Example/sglang_gptoss_120b_example.yaml b/examples/04_GPTOSS120B_Example/sglang_gptoss_120b_example.yaml index 5a2d2050e..2fbcdaa88 100644 --- a/examples/04_GPTOSS120B_Example/sglang_gptoss_120b_example.yaml +++ b/examples/04_GPTOSS120B_Example/sglang_gptoss_120b_example.yaml @@ -1,7 +1,6 @@ name: "gpt-oss-120b-benchmark" version: "1.0" type: "online" -timeout: 60 model_params: name: "openai/gpt-oss-120b" @@ -39,11 +38,13 @@ datasets: num_repeats: 5 settings: runtime: - min_duration_ms: 3000 - max_duration_ms: 60000 scheduler_random_seed: 42 dataloader_random_seed: 42 + timeouts: + min_duration_ms: 3000 + max_duration_ms: 60000 + load_pattern: type: "concurrency" target_concurrency: 512 diff --git a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_example.yaml b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_example.yaml index 2780e0b49..02fa785b9 100644 --- a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_example.yaml +++ b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_example.yaml @@ -1,7 +1,6 @@ name: "gpt-oss-120b-benchmark" version: "1.0" type: "online" -timeout: 60 model_params: name: "openai/gpt-oss-120b" @@ -42,11 +41,13 @@ datasets: settings: runtime: - min_duration_ms: 3000 - max_duration_ms: 60000 scheduler_random_seed: 42 dataloader_random_seed: 42 + timeouts: + min_duration_ms: 3000 + max_duration_ms: 60000 + load_pattern: type: "concurrency" target_concurrency: 512 diff --git a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml index 2a944a00b..04f8d227c 100644 --- a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml +++ b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml @@ -16,7 +16,6 @@ name: "gpt-oss-120b-per-dataset-osl" version: "1.0" type: "online" -timeout: 9000 model_params: name: "openai/gpt-oss-120b" @@ -64,12 +63,14 @@ datasets: settings: runtime: - min_duration_ms: 30000 - max_duration_ms: 14400000 # 4h whole-session deadline (perf + all accuracy phases) scheduler_random_seed: 42 dataloader_random_seed: 42 n_samples_to_issue: 2000 # PERF phase only; accuracy phases issue their own sample counts + timeouts: + min_duration_ms: 30000 + max_duration_ms: 14400000 # 4h cap on the performance phase only + load_pattern: type: "concurrency" target_concurrency: 1024 diff --git a/examples/05_Llama3.1-8B_Example/offline_llama3_8b_cnn.yaml b/examples/05_Llama3.1-8B_Example/offline_llama3_8b_cnn.yaml index 57e105c76..064c0bfc8 100644 --- a/examples/05_Llama3.1-8B_Example/offline_llama3_8b_cnn.yaml +++ b/examples/05_Llama3.1-8B_Example/offline_llama3_8b_cnn.yaml @@ -28,12 +28,14 @@ datasets: settings: runtime: - min_duration_ms: 60000 # 1 minute - max_duration_ms: 360000 # 6 minutes (Arbitrary here, and doesn't have counterpart in legacy loadgen) scheduler_random_seed: 137 # For Poisson/distribution sampling dataloader_random_seed: 111 # For dataset shuffling (Will be updated after rng seeds are finalized for submission) n_samples_to_issue: 13368 # Number of samples to issue (for offline, this should match the dataset samples) + timeouts: + min_duration_ms: 60000 # 1 minute + max_duration_ms: 360000 # 6 minutes (Arbitrary here, and doesn't have counterpart in legacy loadgen) + load_pattern: type: "max_throughput" diff --git a/examples/05_Llama3.1-8B_Example/online_llama3_8b_cnn.yaml b/examples/05_Llama3.1-8B_Example/online_llama3_8b_cnn.yaml index 66861f2f5..8031407dc 100644 --- a/examples/05_Llama3.1-8B_Example/online_llama3_8b_cnn.yaml +++ b/examples/05_Llama3.1-8B_Example/online_llama3_8b_cnn.yaml @@ -28,12 +28,14 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minutes - max_duration_ms: 3600000 # 60 minutes (Arbitrary here, and doesn't have counterpart in legacy loadgen) scheduler_random_seed: 137 # For Poisson/distribution sampling dataloader_random_seed: 111 # For dataset shuffling (Will be updated after rng seeds are finalized for submission) n_samples_to_issue: 13368 + timeouts: + min_duration_ms: 600000 # 10 minutes + max_duration_ms: 3600000 # 60 minutes (Arbitrary here, and doesn't have counterpart in legacy loadgen) + load_pattern: type: "concurrency" target_concurrency: 128 diff --git a/examples/06_Llama2-70B_Example/online_llama2_70b_orca.yaml b/examples/06_Llama2-70B_Example/online_llama2_70b_orca.yaml index 5a7f6ce53..ceadd8562 100644 --- a/examples/06_Llama2-70B_Example/online_llama2_70b_orca.yaml +++ b/examples/06_Llama2-70B_Example/online_llama2_70b_orca.yaml @@ -22,11 +22,13 @@ datasets: settings: runtime: - min_duration_ms: 60000 # 1 minute - max_duration_ms: 600000 # 10 minutes scheduler_random_seed: 42 # For Poisson/distribution sampling dataloader_random_seed: 42 # For dataset shuffling + timeouts: + min_duration_ms: 60000 # 1 minute + max_duration_ms: 600000 # 10 minutes + load_pattern: type: "poisson" target_qps: 10 diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml index 8a3438af8..e07c4f89f 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/interactive_qwen3_vl_235b_a22b_shopify_8k.yaml @@ -3,7 +3,6 @@ name: "interactive-qwen3-vl-235b-a22b-shopify-8k-benchmark" version: "1.0" type: "online" -timeout: 1800 # 30 minutes for quick interactive runs model_params: name: "Qwen/Qwen3-VL-235B-A22B-Instruct" @@ -24,10 +23,12 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minute scheduler_random_seed: 42 dataloader_random_seed: 42 + timeouts: + min_duration_ms: 600000 # 10 minute + load_pattern: type: "poisson" target_qps: 3 diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml index e830242d8..c44f715b2 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml @@ -3,7 +3,6 @@ name: "offline-qwen3-vl-235b-a22b-shopify-benchmark" version: "1.0" type: "offline" -timeout: 14400 # Perf + acc run takes over 3 hours, consider limit n_samples_to_issue for perf run or remove accuracy dataset to skip accuracy run model_params: name: "Qwen/Qwen3-VL-235B-A22B-Instruct" @@ -23,10 +22,15 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minutes, this is override when n_samples_to_issue is set scheduler_random_seed: 42 # For Poisson/distribution sampling dataloader_random_seed: 42 # For dataset shuffling + timeouts: + min_duration_ms: 600000 # 10 minutes, this is override when n_samples_to_issue is set + warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) + performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) + accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) + load_pattern: type: "max_throughput" @@ -37,12 +41,8 @@ settings: recv_buffer_size: 16777216 send_buffer_size: 16777216 max_connections: 1000 - # Increase timeout for slow worker startup (spawn, imports). Default 40s may be too short. + # Increase timeout for slow worker startup (spawn, imports). Default 60s may be too short. worker_initialization_timeout: 120 - drain: - warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) - accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) warmup: enabled: true # Enable warmup phase before performance run n_requests: 1600 # Warmup request count (None = full dataset once) diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml index a14183d20..2c3a653c4 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml @@ -2,7 +2,6 @@ name: "online-qwen3-vl-235b-a22b-shopify-benchmark" version: "1.0" type: "online" -timeout: 14400 model_params: name: "Qwen/Qwen3-VL-235B-A22B-Instruct" @@ -23,10 +22,12 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minutes, this is override when n_samples_to_issue is set scheduler_random_seed: 42 dataloader_random_seed: 42 + timeouts: + min_duration_ms: 600000 # 10 minutes, this is override when n_samples_to_issue is set + load_pattern: type: "poisson" target_qps: 5 @@ -38,7 +39,7 @@ settings: recv_buffer_size: 16777216 send_buffer_size: 16777216 max_connections: 1000 - # Increase timeout for slow worker startup (spawn, imports). Default 40s may be too short. + # Increase timeout for slow worker startup (spawn, imports). Default 60s may be too short. worker_initialization_timeout: 120 warmup: enabled: true # Enable warmup phase before performance run diff --git a/examples/09_Wan22_VideoGen_Example/offline_wan22.yaml b/examples/09_Wan22_VideoGen_Example/offline_wan22.yaml index 508af97c8..ee3742cc0 100644 --- a/examples/09_Wan22_VideoGen_Example/offline_wan22.yaml +++ b/examples/09_Wan22_VideoGen_Example/offline_wan22.yaml @@ -34,11 +34,13 @@ datasets: settings: runtime: - max_duration_ms: 600000 # 10 minute cap scheduler_random_seed: 42 dataloader_random_seed: 42 n_samples_to_issue: 248 + timeouts: + max_duration_ms: 600000 # 10 minute cap + load_pattern: type: "max_throughput" diff --git a/examples/09_Wan22_VideoGen_Example/offline_wan22_accuracy.yaml b/examples/09_Wan22_VideoGen_Example/offline_wan22_accuracy.yaml index b3a4dd50e..684f0d2c9 100644 --- a/examples/09_Wan22_VideoGen_Example/offline_wan22_accuracy.yaml +++ b/examples/09_Wan22_VideoGen_Example/offline_wan22_accuracy.yaml @@ -51,11 +51,13 @@ datasets: settings: runtime: - max_duration_ms: 600000 scheduler_random_seed: 42 dataloader_random_seed: 42 n_samples_to_issue: 248 + timeouts: + max_duration_ms: 600000 + load_pattern: type: "max_throughput" diff --git a/examples/09_Wan22_VideoGen_Example/offline_wan22_submission.yaml b/examples/09_Wan22_VideoGen_Example/offline_wan22_submission.yaml index af50258b8..9e9a4aad4 100644 --- a/examples/09_Wan22_VideoGen_Example/offline_wan22_submission.yaml +++ b/examples/09_Wan22_VideoGen_Example/offline_wan22_submission.yaml @@ -50,13 +50,15 @@ audit: settings: runtime: + scheduler_random_seed: 42 + dataloader_random_seed: 42 + n_samples_to_issue: 144 # perf run only; accuracy scores its dataset (248), audit uses audit.samples + + timeouts: # NOTE: runs are count-driven (n_samples_to_issue / audit.samples). min_duration_ms is # NOT enforced as a duration floor by the current stop logic (counts take priority); # MLCommons' 10-min minimum / AND-semantics is future work. Only max_duration_ms caps. max_duration_ms: 14400000 # 4-hour ceiling - scheduler_random_seed: 42 - dataloader_random_seed: 42 - n_samples_to_issue: 144 # perf run only; accuracy scores its dataset (248), audit uses audit.samples load_pattern: type: "max_throughput" diff --git a/examples/09_Wan22_VideoGen_Example/single_stream_wan22_submission.yaml b/examples/09_Wan22_VideoGen_Example/single_stream_wan22_submission.yaml index bec6f8720..e8d4209c3 100644 --- a/examples/09_Wan22_VideoGen_Example/single_stream_wan22_submission.yaml +++ b/examples/09_Wan22_VideoGen_Example/single_stream_wan22_submission.yaml @@ -50,13 +50,15 @@ audit: settings: runtime: + scheduler_random_seed: 42 + dataloader_random_seed: 42 + n_samples_to_issue: 20 # perf run only; accuracy scores its dataset (248), audit uses its own counts + + timeouts: # NOTE: runs are count-driven (n_samples_to_issue / audit counts). min_duration_ms is # NOT enforced as a duration floor by the current stop logic (counts take priority); # MLCommons' 10-min minimum / AND-semantics is future work. Only max_duration_ms caps. max_duration_ms: 7200000 # 2-hour ceiling - scheduler_random_seed: 42 - dataloader_random_seed: 42 - n_samples_to_issue: 20 # perf run only; accuracy scores its dataset (248), audit uses its own counts load_pattern: type: "concurrency" diff --git a/examples/10_Agentic_Inference/README.md b/examples/10_Agentic_Inference/README.md index ab3673b51..a568bb2f4 100644 --- a/examples/10_Agentic_Inference/README.md +++ b/examples/10_Agentic_Inference/README.md @@ -102,7 +102,7 @@ The runnable config is this field is `false`, the client keeps replaying already-started active trajectories to completion for accuracy/log coverage, but those later-issued turns are outside the performance measurement window. -- `settings.runtime.min_duration_ms`: minimum run duration. Agentic inference replay +- `settings.timeouts.min_duration_ms`: minimum run duration. Agentic inference replay completion is controlled by trajectory budget and active conversation drain. - `settings.load_pattern.type`: enables conversation-aware issuing. - `settings.load_pattern.target_concurrency`: maximum active conversations. Each @@ -129,7 +129,7 @@ For official Kimi agentic benchmark runs, keep these values fixed: - `model_params.chat_template_kwargs.preserve_thinking: true` - First dataset `type: performance` - First dataset `accuracy_config.eval_method: agentic_inference_inline` -- `settings.runtime.min_duration_ms: 0` +- `settings.timeouts.min_duration_ms: 0` - `settings.load_pattern.type: agentic_inference` - `settings.client.warmup_connections: 0` - `settings.client.max_idle_time: 0.5` diff --git a/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml b/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml index 9740aa4c1..ad4318bdf 100644 --- a/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml +++ b/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml @@ -25,7 +25,7 @@ datasets: stop_issuing_on_first_user_complete: false settings: - runtime: + timeouts: min_duration_ms: 0 load_pattern: diff --git a/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml b/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml index e8bd1e418..f20daf8a9 100644 --- a/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml +++ b/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml @@ -14,9 +14,7 @@ # Phases run perf -> accuracy (framework order). Both are deterministic # (temperature 0, seed 42) against a reasoning-off server, so order does not # affect results. Total wall-clock ~5.5 h on a single-stream edge box -# (e.g. NVIDIA Jetson AGX Thor, Qwen3.6-27B Q4_K_M, reasoning off); `timeout` -# below is sized to 6 h. -# +# (e.g. NVIDIA Jetson AGX Thor, Qwen3.6-27B Q4_K_M, reasoning off). # Requires: pip install -e ".[bfcl]" (the BFCL accuracy dataset pulls bfcl-eval) # # Run from the repo root (start the edge server first; see README.md). The @@ -33,7 +31,6 @@ name: "edge-agentic-full-run" version: "1.0" type: "online" -timeout: 21600 # 6 h: ~2.5 h perf + ~3 h accuracy, with headroom. model_params: name: "Qwen3.6-27B-Q4_K_M" # set to your served model name. @@ -90,11 +87,12 @@ datasets: settings: runtime: + dataloader_random_seed: 42 # BFCL accuracy sampling seed; compliance requires 42. + timeouts: min_duration_ms: 0 # Safety cap (4 h) so the performance phase stays bounded even if decode is # slower than expected; one pass should finish in ~2.5 h on an edge box. max_duration_ms: 14400000 - dataloader_random_seed: 42 # BFCL accuracy sampling seed; compliance requires 42. load_pattern: # Drives the performance phase; the accuracy phase self-overrides to # max_throughput. diff --git a/scripts/regenerate_templates.py b/scripts/regenerate_templates.py index 6cb5c4358..87e2aec40 100644 --- a/scripts/regenerate_templates.py +++ b/scripts/regenerate_templates.py @@ -357,9 +357,11 @@ def _build_minimal(test_type: TestType, overrides: dict) -> dict: "datasets": [PERF_DATASET], "settings": { "runtime": { + "n_samples_to_issue": None, + }, + "timeouts": { "min_duration_ms": 600000, "max_duration_ms": 0, - "n_samples_to_issue": None, }, }, "endpoint_config": {"endpoints": [PLACEHOLDER_ENDPOINT]}, diff --git a/src/inference_endpoint/async_utils/services/launcher.py b/src/inference_endpoint/async_utils/services/launcher.py index 9e53c19c4..b710b8bf0 100644 --- a/src/inference_endpoint/async_utils/services/launcher.py +++ b/src/inference_endpoint/async_utils/services/launcher.py @@ -151,6 +151,17 @@ def kill_all(self) -> None: if proc.poll() is None: proc.kill() + def terminate_all(self) -> None: + """SIGTERM all managed subprocesses (graceful; see kill_all for SIGKILL). + + The metrics aggregator installs a SIGTERM handler that writes an + INTERRUPTED final snapshot before exiting — this is the abort path + for the whole-run watchdog. + """ + for proc in self._procs: + if proc.poll() is None: + proc.terminate() + def wait_for_exit(self, timeout: float | None = 60.0) -> None: """Wait for all service subprocesses to exit. diff --git a/src/inference_endpoint/commands/benchmark/cli.py b/src/inference_endpoint/commands/benchmark/cli.py index 0edb4da79..68aaa0f1e 100644 --- a/src/inference_endpoint/commands/benchmark/cli.py +++ b/src/inference_endpoint/commands/benchmark/cli.py @@ -174,7 +174,15 @@ def from_config( except (yaml.YAMLError, ValidationError, ValueError, FileNotFoundError) as e: raise InputValidationError(f"Config error: {e}") from e if timeout is not None: - resolved = resolved.with_updates(timeout=timeout) + resolved = resolved.with_updates( + settings=resolved.settings.model_copy( + update={ + "timeouts": resolved.settings.timeouts.with_updates( + run_timeout_s=timeout + ) + } + ) + ) if report_dir is not None: resolved = resolved.with_updates(report_dir=report_dir) test_mode = mode or ( diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index dcb3275bb..ef9b1126e 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -160,6 +160,9 @@ class BenchmarkResult: # settings.profiling.engine is set; None otherwise. Rendered into # report.txt and a sibling profiling.json by finalize_benchmark. profiling: dict[str, Any] | None = None + # True when the whole-run watchdog (settings.timeouts.run_timeout_s) fired. + # The report is INTERRUPTED and run_benchmark exits non-zero. + run_timed_out: bool = False @dataclass @@ -517,7 +520,7 @@ def _build_phases( ) -> list[PhaseConfig]: """Build the phase list from BenchmarkContext.""" phases: list[PhaseConfig] = [] - drain_cfg = ctx.config.settings.drain + timeouts_cfg = ctx.config.settings.timeouts if ctx.dataloader is not None and ctx.rt_settings is not None: warmup_cfg = ctx.config.settings.warmup @@ -547,7 +550,7 @@ def _build_phases( warmup_dataset, PhaseType.WARMUP, drain_after=warmup_cfg.drain, - drain_timeout=drain_cfg.warmup_timeout_s, + drain_timeout=timeouts_cfg.warmup_drain_timeout_s, ) ) @@ -558,7 +561,7 @@ def _build_phases( ctx.dataloader, PhaseType.PERFORMANCE, strategy=perf_strategy, - drain_timeout=drain_cfg.performance_timeout_s, + drain_timeout=timeouts_cfg.performance_drain_timeout_s, ) ) @@ -610,7 +613,7 @@ def _build_phases( acc_settings, acc_ds, PhaseType.ACCURACY, - drain_timeout=drain_cfg.accuracy_timeout_s, + drain_timeout=timeouts_cfg.accuracy_drain_timeout_s, ) ) @@ -849,13 +852,19 @@ async def _run_benchmark_async( aggregator_args.append("--streaming") if ctx.tokenizer_name is not None: aggregator_args.extend(["--tokenizer", ctx.tokenizer_name]) + # Aggregator argv contract keeps 0 = unlimited (hand-launch + # default); the schema uses None = unlimited, so convert here. + metrics_drain_s = config.settings.timeouts.metrics_drain_timeout_s aggregator_args.extend( - ["--drain-timeout", str(config.settings.drain.metrics_drain_timeout_s)] + [ + "--drain-timeout", + str(metrics_drain_s if metrics_drain_s is not None else 0.0), + ] ) aggregator_args.extend( [ "--tokenizer-workers", - str(config.settings.drain.metrics_tokenizer_workers), + str(config.settings.metrics.tokenizer_workers), ] ) @@ -882,7 +891,7 @@ async def _run_benchmark_async( args=event_logger_args, ), ], - timeout=config.settings.service_ready_timeout_s, + timeout=config.settings.timeouts.service_ready_timeout_s, ) # Create endpoint client on the shared loop @@ -1003,8 +1012,25 @@ def _on_sample_complete(result: QueryResult) -> None: profile_endpoints, profiling_cfg.engine, "stop" ) session_completed_normally = False - - def _on_global_timeout() -> None: + run_timed_out = False + run_timeout_s = config.settings.timeouts.run_timeout_s + + def _on_run_timeout() -> None: + nonlocal run_timed_out + run_timed_out = True + logger.error( + "Run timeout (%.1fs) reached; aborting run — report will " + "be marked INTERRUPTED.", + run_timeout_s, + ) + # SIGTERM services first: the aggregator's handler writes the + # INTERRUPTED final snapshot. publish_final is idempotent, so + # the ENDED-driven finalize after session.stop() is a no-op — + # the run can never be reported COMPLETE once this fires. + launcher.terminate_all() + session.stop() + + def _on_perf_phase_timeout() -> None: if not _timeout_done: logger.warning( "Performance phase max_duration reached (%d ms); " @@ -1015,7 +1041,14 @@ def _on_global_timeout() -> None: # perf+accuracy run still runs accuracy after the perf cap. session.stop_current_phase() - perf_timeout = _PerfPhaseTimeout(loop, max_duration_ms, _on_global_timeout) + perf_timeout = _PerfPhaseTimeout( + loop, max_duration_ms, _on_perf_phase_timeout + ) + run_watchdog = ( + loop.call_later(run_timeout_s, _on_run_timeout) + if run_timeout_s is not None + else None + ) def _on_phase_start(phase: PhaseConfig) -> None: # _PerfPhaseTimeout arms the perf cap on PERFORMANCE and cancels it @@ -1045,10 +1078,20 @@ def _on_phase_start(phase: PhaseConfig) -> None: result = await session.run(phases, on_phase_start=_on_phase_start) session_completed_normally = True except Exception as e: + if run_timed_out: + # The watchdog SIGTERMed the services before stopping the + # session; a teardown race can surface here as a generic + # exception. Report the timeout, not the symptom. + raise ExecutionError( + f"Run timeout ({run_timeout_s}s) reached; run aborted " + "and report marked INTERRUPTED" + ) from e raise ExecutionError(f"Benchmark execution failed: {e}") from e finally: _timeout_done = True perf_timeout.cancel() + if run_watchdog is not None: + run_watchdog.cancel() loop.remove_signal_handler(signal.SIGINT) # Fire /stop_profile for URLs whose /start_profile succeeded. # Unifies the clean phase-end path and the abort path — @@ -1179,6 +1222,7 @@ def _on_phase_start(phase: PhaseConfig) -> None: report=report, tmpfs_dir=tmpfs_dir, profiling=profiling_payload, + run_timed_out=run_timed_out, ) @@ -1400,6 +1444,11 @@ def run_benchmark( try: bench = run_benchmark_async(ctx) finalize_benchmark(ctx, bench) + if bench.run_timed_out: + raise ExecutionError( + f"Run timeout ({config.settings.timeouts.run_timeout_s}s) " + "reached; run aborted and report marked INTERRUPTED" + ) except KeyboardInterrupt: # Salvage results (finally), then propagate to main.py -> exit 130. logger.warning("Benchmark interrupted by user") diff --git a/src/inference_endpoint/config/runtime_settings.py b/src/inference_endpoint/config/runtime_settings.py index 6573ef82a..c51d51bfc 100644 --- a/src/inference_endpoint/config/runtime_settings.py +++ b/src/inference_endpoint/config/runtime_settings.py @@ -174,6 +174,7 @@ def _from_config_default( """ # Extract settings from immutable Pydantic models runtime_cfg = config.settings.runtime + timeouts_cfg = config.settings.timeouts load_pattern_cfg = config.settings.load_pattern # TODO: The default target_qps should be None in Offline mode, but we use 10.0 for now. @@ -188,10 +189,10 @@ def _from_config_default( kwargs = { "metric_target": metrics.Throughput(effective_qps), "reported_metrics": [metrics.Throughput(effective_qps)], - "min_duration_ms": runtime_cfg.min_duration_ms, + "min_duration_ms": timeouts_cfg.min_duration_ms, "max_duration_ms": None - if runtime_cfg.max_duration_ms == 0 - else runtime_cfg.max_duration_ms, + if timeouts_cfg.max_duration_ms == 0 + else timeouts_cfg.max_duration_ms, "n_samples_from_dataset": dataloader_num_samples, "n_samples_to_issue": runtime_cfg.n_samples_to_issue, # From config (CLI --num-samples or YAML) "min_sample_count": 1, diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index b96843fea..adcfa6449 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -49,6 +49,7 @@ from ..exceptions import CLIError from ..utils import WithUpdatesMixin from .ruleset_base import BenchmarkSuiteRuleset +from .timeouts import Timeouts from .utils import parse_dataset_string, resolve_env_vars logger = logging.getLogger(__name__) @@ -602,36 +603,12 @@ class RuntimeConfig(BaseModel): 1. n_samples_to_issue (if specified) — explicit override 2. Calculated from QPS * duration — duration-based (default: 600000ms) 3. All dataset samples — fallback when duration is 0 + + Durations live in ``settings.timeouts`` (see ``config/timeouts.py``). """ model_config = ConfigDict(extra="forbid", frozen=True) - min_duration_ms: Annotated[ - int, - cyclopts.Parameter( - alias="--duration", help="Min duration (ms, or with suffix: 600s, 10m)" - ), - ] = Field(600000, ge=0) - max_duration_ms: int = Field( - 0, - ge=0, - description="Maximum test duration in ms (0 for no limit)", - ) - - @field_validator("min_duration_ms", "max_duration_ms", mode="before") - @classmethod - def _parse_duration_suffix(cls, v: object) -> object: - """Accept duration with unit suffix: 600s, 10m, 600000ms, or plain int (ms).""" - if isinstance(v, str): - v = v.strip() - if v.endswith("ms"): - return int(v[:-2]) - if v.endswith("m"): - return int(float(v[:-1]) * 60_000) - if v.endswith("s"): - return int(float(v[:-1]) * 1000) - return v - n_samples_to_issue: Annotated[ int | None, cyclopts.Parameter(alias="--num-samples", help="Sample count override"), @@ -639,15 +616,6 @@ def _parse_duration_suffix(cls, v: object) -> object: scheduler_random_seed: int = Field(42, description="Scheduler RNG seed") dataloader_random_seed: int = Field(42, description="Dataloader RNG seed") - @model_validator(mode="after") - def _validate_durations(self) -> Self: - if self.max_duration_ms != 0 and self.max_duration_ms < self.min_duration_ms: - raise ValueError( - f"max_duration_ms ({self.max_duration_ms}) must be >= " - f"min_duration_ms ({self.min_duration_ms})" - ) - return self - @cyclopts.Parameter(name="*") class LoadPattern(BaseModel): @@ -780,65 +748,12 @@ class WarmupConfig(BaseModel): ] = Field(42, description="RNG seed for warmup scheduling and sample ordering") -class DrainConfig(BaseModel): - """Per-phase in-flight response drain timeout configuration.""" +class MetricsConfig(BaseModel): + """Metrics-aggregator tuning knobs (non-timeout; deadlines live in Timeouts).""" model_config = ConfigDict(extra="forbid", frozen=True) - warmup_timeout_s: Annotated[ - float | None, - cyclopts.Parameter( - alias="--warmup-drain-timeout", - help="Warmup drain timeout in seconds (None = wait indefinitely)", - ), - ] = Field( - 240.0, - gt=0, - description="Warmup drain timeout in seconds (None = wait indefinitely)", - ) - performance_timeout_s: Annotated[ - float | None, - cyclopts.Parameter( - alias="--performance-drain-timeout", - help="Performance drain timeout in seconds (None = wait indefinitely)", - ), - ] = Field( - 240.0, - gt=0, - description="Performance drain timeout in seconds (None = wait indefinitely)", - ) - accuracy_timeout_s: Annotated[ - float | None, - cyclopts.Parameter( - alias="--accuracy-drain-timeout", - help="Accuracy drain timeout in seconds (None = wait indefinitely)", - ), - ] = Field( - None, - gt=0, - description="Accuracy drain timeout in seconds (None = wait indefinitely)", - ) - metrics_drain_timeout_s: Annotated[ - float, - cyclopts.Parameter( - alias="--metrics-drain-timeout", - help=( - "Wall-clock budget (seconds) for the metrics aggregator to finish " - "tokenizing buffered samples after the run ends. Set to 0 to wait " - "indefinitely. Increase for very large datasets where the end-of-run " - "tokenize batch is big." - ), - ), - ] = Field( - 0.0, - ge=0, - description=( - "Wall-clock budget (seconds) to finish tokenizing buffered samples " - "after ENDED (default: 0 = unlimited). An incomplete drain is " - "surfaced via n_pending_tasks > 0, never silently dropped." - ), - ) - metrics_tokenizer_workers: Annotated[ + tokenizer_workers: Annotated[ int, cyclopts.Parameter( alias="--metrics-tokenizer-workers", @@ -933,23 +848,13 @@ class Settings(BaseModel): runtime: RuntimeConfig = Field(default_factory=RuntimeConfig) load_pattern: LoadPattern = Field(default_factory=LoadPattern) client: HTTPClientConfig = Field(default_factory=HTTPClientConfig) - drain: DrainConfig = Field( - default_factory=DrainConfig, - description="Per-phase in-flight response drain timeout configuration", + timeouts: Timeouts = Field( + default_factory=Timeouts, + description="All global durations and deadlines (see config/timeouts.py)", ) + metrics: MetricsConfig = Field(default_factory=MetricsConfig) warmup: WarmupConfig = Field(default_factory=WarmupConfig) profiling: ProfilingConfig = Field(default_factory=ProfilingConfig) - service_ready_timeout_s: Annotated[ - float, - cyclopts.Parameter( - alias="--service-ready-timeout", - help="Seconds to wait for metrics/event-logger services to start", - ), - ] = Field( - default=30.0, - ge=0, - description="Seconds to wait for metrics-aggregator/event-logger services to become ready.", - ) class OfflineSettings(Settings): @@ -1040,10 +945,6 @@ class BenchmarkConfig(WithUpdatesMixin, BaseModel): Path | None, cyclopts.Parameter(alias="--report-dir", help="Report output directory"), ] = None - timeout: Annotated[ - float | None, - cyclopts.Parameter(alias="--timeout", help="Global timeout in seconds"), - ] = None # verbose is handled by cyclopts meta app (-v flag), not here verbose: Annotated[bool, cyclopts.Parameter(show=False)] = Field( False, description="Enable verbose logging" @@ -1082,7 +983,6 @@ def _resolve_and_validate(self) -> Self: Validation: - Workers must be -1 (auto) or >= 1 - - max_duration_ms >= min_duration_ms >= 0 - No duplicate dataset (name, type) pairs - Load pattern must match test type """ diff --git a/src/inference_endpoint/config/templates/concurrency_template.yaml b/src/inference_endpoint/config/templates/concurrency_template.yaml index 2f38e4618..7a316f0af 100644 --- a/src/inference_endpoint/config/templates/concurrency_template.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template.yaml @@ -10,9 +10,10 @@ datasets: # Dataset configs prompt: text_input settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) n_samples_to_issue: null # Sample count override + timeouts: # All global durations and deadlines (see config/timeouts.py) + min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) + max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) load_pattern: type: concurrency # Load pattern type | options: max_throughput, poisson, concurrency, agentic_inference, burst, step target_concurrency: 32 # Concurrent requests diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index 521f6f335..caf741fa7 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -51,8 +51,6 @@ datasets: # Dataset configs generation_config_override: null # Per-dataset overrides for the top-level model_params (sparse — only the fields you want to override). Merged on top of BenchmarkConfig.model_params at dataset-load time. Useful for MLPerf-style runs where accuracy and performance use different output budgets in the same fleet, e.g. generation_config_override: {max_new_tokens: 32768, temperature: 0.0}. NOTE: per-run/identity keys (`name`, `streaming`, `tokenizer_name`) are rejected here — set them on top-level model_params. settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) n_samples_to_issue: null # Sample count override scheduler_random_seed: 42 # Scheduler RNG seed dataloader_random_seed: 42 # Dataloader RNG seed @@ -82,29 +80,32 @@ settings: max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system - drain: - warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) - accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) - metrics_drain_timeout_s: 0.0 # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (default: 0 = unlimited). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. - metrics_tokenizer_workers: 2 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 2; 0 = defer everything to the end-of-run drain). + timeouts: # All global durations and deadlines (see config/timeouts.py) + min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) + max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) + run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. + service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. + warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) + performance_drain_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) + accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) + metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. + metrics: + tokenizer_workers: 2 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 2; 0 = defer everything to the end-of-run drain). warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) salt: true # Prepend a unique random hex salt to each warmup prompt - drain: false + drain: false # Drain in-flight warmup requests before starting the performance phase warmup_random_seed: 42 # RNG seed for warmup scheduling and sample ordering profiling: engine: null # Profile the named inference engine around the performance phase | options: vllm urls: null # URL(s) the profiler start/stop triggers are derived from. When None, derived from endpoint_config.endpoints instead. Use when the profiler admin endpoint differs from the inference endpoint. - service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen report_dir: null # Report output directory -timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning audit: null # Compliance audit config (YAML only). When set, runs the audit after the main benchmark. diff --git a/src/inference_endpoint/config/templates/offline_template.yaml b/src/inference_endpoint/config/templates/offline_template.yaml index d672491cf..c1a0db1c1 100644 --- a/src/inference_endpoint/config/templates/offline_template.yaml +++ b/src/inference_endpoint/config/templates/offline_template.yaml @@ -10,9 +10,10 @@ datasets: # Dataset configs prompt: text_input settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) n_samples_to_issue: null # Sample count override + timeouts: # All global durations and deadlines (see config/timeouts.py) + min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) + max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index 51e06c738..52530a3ca 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -51,8 +51,6 @@ datasets: # Dataset configs generation_config_override: null # Per-dataset overrides for the top-level model_params (sparse — only the fields you want to override). Merged on top of BenchmarkConfig.model_params at dataset-load time. Useful for MLPerf-style runs where accuracy and performance use different output budgets in the same fleet, e.g. generation_config_override: {max_new_tokens: 32768, temperature: 0.0}. NOTE: per-run/identity keys (`name`, `streaming`, `tokenizer_name`) are rejected here — set them on top-level model_params. settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) n_samples_to_issue: null # Sample count override scheduler_random_seed: 42 # Scheduler RNG seed dataloader_random_seed: 42 # Dataloader RNG seed @@ -82,29 +80,32 @@ settings: max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system - drain: - warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) - accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) - metrics_drain_timeout_s: 0.0 # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (default: 0 = unlimited). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. - metrics_tokenizer_workers: 2 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 2; 0 = defer everything to the end-of-run drain). + timeouts: # All global durations and deadlines (see config/timeouts.py) + min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) + max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) + run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. + service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. + warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) + performance_drain_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) + accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) + metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. + metrics: + tokenizer_workers: 2 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 2; 0 = defer everything to the end-of-run drain). warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) salt: true # Prepend a unique random hex salt to each warmup prompt - drain: false + drain: false # Drain in-flight warmup requests before starting the performance phase warmup_random_seed: 42 # RNG seed for warmup scheduling and sample ordering profiling: engine: null # Profile the named inference engine around the performance phase | options: vllm urls: null # URL(s) the profiler start/stop triggers are derived from. When None, derived from endpoint_config.endpoints instead. Use when the profiler admin endpoint differs from the inference endpoint. - service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen report_dir: null # Report output directory -timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning audit: null # Compliance audit config (YAML only). When set, runs the audit after the main benchmark. diff --git a/src/inference_endpoint/config/templates/online_template.yaml b/src/inference_endpoint/config/templates/online_template.yaml index 26ce8f178..c359e108c 100644 --- a/src/inference_endpoint/config/templates/online_template.yaml +++ b/src/inference_endpoint/config/templates/online_template.yaml @@ -10,9 +10,10 @@ datasets: # Dataset configs prompt: text_input settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) n_samples_to_issue: null # Sample count override + timeouts: # All global durations and deadlines (see config/timeouts.py) + min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) + max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) load_pattern: type: poisson # Load pattern type | options: max_throughput, poisson, concurrency, agentic_inference, burst, step target_qps: 10.0 # Target QPS diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index c29abc23f..e5a6af91a 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -51,8 +51,6 @@ datasets: # Dataset configs generation_config_override: null # Per-dataset overrides for the top-level model_params (sparse — only the fields you want to override). Merged on top of BenchmarkConfig.model_params at dataset-load time. Useful for MLPerf-style runs where accuracy and performance use different output budgets in the same fleet, e.g. generation_config_override: {max_new_tokens: 32768, temperature: 0.0}. NOTE: per-run/identity keys (`name`, `streaming`, `tokenizer_name`) are rejected here — set them on top-level model_params. settings: runtime: - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum test duration in ms (0 for no limit) n_samples_to_issue: null # Sample count override scheduler_random_seed: 42 # Scheduler RNG seed dataloader_random_seed: 42 # Dataloader RNG seed @@ -83,29 +81,32 @@ settings: max_idle_time: 4.0 # Discard connections idle longer than this (seconds) min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system - drain: - warmup_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) - accuracy_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) - metrics_drain_timeout_s: 0.0 # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (default: 0 = unlimited). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. - metrics_tokenizer_workers: 2 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 2; 0 = defer everything to the end-of-run drain). + timeouts: # All global durations and deadlines (see config/timeouts.py) + min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) + max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) + run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. + service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. + warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) + performance_drain_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) + accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) + metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. + metrics: + tokenizer_workers: 2 # In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT (default: 2; 0 = defer everything to the end-of-run drain). warmup: enabled: false # Enable warmup phase before performance run n_requests: null # Warmup request count (None = full dataset once) salt: true # Prepend a unique random hex salt to each warmup prompt - drain: false + drain: false # Drain in-flight warmup requests before starting the performance phase warmup_random_seed: 42 # RNG seed for warmup scheduling and sample ordering profiling: engine: null # Profile the named inference engine around the performance phase | options: vllm urls: null # URL(s) the profiler start/stop triggers are derived from. When None, derived from endpoint_config.endpoints instead. Use when the profiler admin endpoint differs from the inference endpoint. - service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 api_key: null # API key api_type: openai # API type: openai, sglang, or videogen | options: openai, openai_completions, sglang, videogen report_dir: null # Report output directory -timeout: null # Global timeout in seconds verbose: false # Enable verbose logging enable_cpu_affinity: true # NUMA-aware CPU pinning audit: null # Compliance audit config (YAML only). When set, runs the audit after the main benchmark. diff --git a/src/inference_endpoint/config/templates/submission_template.yaml b/src/inference_endpoint/config/templates/submission_template.yaml index ac3c2b11d..459c1b83f 100644 --- a/src/inference_endpoint/config/templates/submission_template.yaml +++ b/src/inference_endpoint/config/templates/submission_template.yaml @@ -46,11 +46,13 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minutes - max_duration_ms: 1800000 # 30 minutes scheduler_random_seed: 42 # For Poisson/distribution sampling dataloader_random_seed: 42 # For dataset shuffling + timeouts: + min_duration_ms: 600000 # 10 minutes + max_duration_ms: 1800000 # 30 minutes + load_pattern: type: "max_throughput" target_qps: 100 diff --git a/src/inference_endpoint/config/timeouts.py b/src/inference_endpoint/config/timeouts.py new file mode 100644 index 000000000..f895ef7e8 --- /dev/null +++ b/src/inference_endpoint/config/timeouts.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""All global time knobs in one frozen model — durations and deadlines. + +Two disjoint categories live here, and they must not be conflated: + +- **Workload durations** (``min_duration_ms``/``max_duration_ms``): part of + the benchmark definition — they shape sample-count math and bound the + performance phase only. Reaching them is a *normal* end of the run. +- **Give-up deadlines** (everything else): failure handling. Reaching one + means something is stuck. ``run_timeout_s`` is the whole-run watchdog: + when it fires the run is aborted and the report is marked INTERRUPTED — + it never derives or caps the per-stage deadlines below it. + +``None`` means "wait indefinitely" for every optional deadline. Deadlines +are resolved to plain floats before the run starts; nothing in the hot +path reads this model. + +Dataset-scoped time knobs (e.g. agentic ``turn_timeout_s``) stay in their +dataset config blocks — they are per-workload behavior, not global. +""" + +from __future__ import annotations + +from typing import Annotated, Self + +import cyclopts +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from ..utils import WithUpdatesMixin + + +@cyclopts.Parameter(name="*") +class Timeouts(WithUpdatesMixin, BaseModel): + """Global durations and deadlines (see module docstring for semantics).""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + # --- Workload durations (benchmark definition, not failure handling) --- + min_duration_ms: Annotated[ + int, + cyclopts.Parameter( + alias="--duration", help="Min duration (ms, or with suffix: 600s, 10m)" + ), + ] = Field(600000, ge=0) + max_duration_ms: int = Field( + 0, + ge=0, + description="Maximum performance-phase duration in ms (0 for no limit)", + ) + + # --- Whole-run watchdog --- + run_timeout_s: Annotated[ + float | None, + cyclopts.Parameter( + alias="--timeout", + help=( + "Whole-run watchdog in seconds (None = off). Firing aborts the " + "run and marks the report INTERRUPTED." + ), + ), + ] = Field( + None, + gt=0, + description=( + "Whole-run watchdog in seconds (None = off). Covers every phase " + "including drains; firing aborts the run, marks the report " + "INTERRUPTED, and exits non-zero. Never derives per-stage deadlines." + ), + ) + + # --- Startup deadlines --- + service_ready_timeout_s: Annotated[ + float, + cyclopts.Parameter( + alias="--service-ready-timeout", + help="Seconds to wait for metrics/event-logger services to start", + ), + ] = Field( + 30.0, + ge=0, + description="Seconds to wait for metrics-aggregator/event-logger services to become ready.", + ) + # --- Drain deadlines (None = wait indefinitely) --- + warmup_drain_timeout_s: Annotated[ + float | None, + cyclopts.Parameter( + alias="--warmup-drain-timeout", + help="Warmup drain timeout in seconds (None = wait indefinitely)", + ), + ] = Field( + 240.0, + gt=0, + description="Warmup drain timeout in seconds (None = wait indefinitely)", + ) + performance_drain_timeout_s: Annotated[ + float | None, + cyclopts.Parameter( + alias="--performance-drain-timeout", + help="Performance drain timeout in seconds (None = wait indefinitely)", + ), + ] = Field( + 240.0, + gt=0, + description="Performance drain timeout in seconds (None = wait indefinitely)", + ) + accuracy_drain_timeout_s: Annotated[ + float | None, + cyclopts.Parameter( + alias="--accuracy-drain-timeout", + help="Accuracy drain timeout in seconds (None = wait indefinitely)", + ), + ] = Field( + None, + gt=0, + description=( + "Accuracy drain timeout in seconds (None = wait indefinitely; " + "accuracy is unbounded by default because every sample must complete)" + ), + ) + metrics_drain_timeout_s: Annotated[ + float | None, + cyclopts.Parameter( + alias="--metrics-drain-timeout", + help=( + "Wall-clock budget (seconds) for the metrics aggregator to finish " + "tokenizing buffered samples after the run ends " + "(None = wait indefinitely)" + ), + ), + ] = Field( + None, + gt=0, + description=( + "Wall-clock budget (seconds) to finish tokenizing buffered samples " + "after ENDED (None = wait indefinitely). An incomplete drain is " + "surfaced via n_pending_tasks > 0, never silently dropped." + ), + ) + + @field_validator("min_duration_ms", "max_duration_ms", mode="before") + @classmethod + def _parse_duration_suffix(cls, v: object) -> object: + """Accept duration with unit suffix: 600s, 10m, 600000ms, or plain int (ms).""" + if isinstance(v, str): + v = v.strip() + if v.endswith("ms"): + return int(v[:-2]) + if v.endswith("m"): + return int(float(v[:-1]) * 60_000) + if v.endswith("s"): + return int(float(v[:-1]) * 1000) + return v + + @model_validator(mode="after") + def _validate_durations(self) -> Self: + if self.max_duration_ms != 0 and self.max_duration_ms < self.min_duration_ms: + raise ValueError( + f"max_duration_ms ({self.max_duration_ms}) must be >= " + f"min_duration_ms ({self.min_duration_ms})" + ) + return self diff --git a/tests/integration/commands/test_accuracy_pipeline.py b/tests/integration/commands/test_accuracy_pipeline.py index 8574599f4..d0eb3ec73 100644 --- a/tests/integration/commands/test_accuracy_pipeline.py +++ b/tests/integration/commands/test_accuracy_pipeline.py @@ -35,13 +35,13 @@ LoadPattern, LoadPatternType, ModelParams, - RuntimeConfig, Settings, StreamingMode, TestMode, TestType, ) from inference_endpoint.config.schema import Dataset as DatasetConfig +from inference_endpoint.config.timeouts import Timeouts from inference_endpoint.endpoint_client.config import HTTPClientConfig @@ -119,7 +119,7 @@ def test_accuracy_scoring_with_echo_server( ), ], settings=Settings( - runtime=RuntimeConfig(min_duration_ms=0), + timeouts=Timeouts(min_duration_ms=0), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig( num_workers=1, warmup_connections=0, max_connections=10 diff --git a/tests/integration/commands/test_benchmark_command.py b/tests/integration/commands/test_benchmark_command.py index 51378695d..04d7c9fc8 100644 --- a/tests/integration/commands/test_benchmark_command.py +++ b/tests/integration/commands/test_benchmark_command.py @@ -33,16 +33,16 @@ LoadPattern, LoadPatternType, ModelParams, - RuntimeConfig, Settings, StreamingMode, TestMode, TestType, ) +from inference_endpoint.config.timeouts import Timeouts from inference_endpoint.endpoint_client.config import HTTPClientConfig _TEST_SETTINGS = Settings( - runtime=RuntimeConfig(min_duration_ms=0), + timeouts=Timeouts(min_duration_ms=0), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig(num_workers=1, warmup_connections=0, max_connections=10), ) @@ -62,7 +62,7 @@ def _config(endpoint_url: str, dataset_path: str, **overrides) -> BenchmarkConfi def _poisson_settings(target_qps: float, duration_s: int = 2) -> Settings: return Settings( - runtime=RuntimeConfig(min_duration_ms=duration_s * 1000), + timeouts=Timeouts(min_duration_ms=duration_s * 1000), load_pattern=LoadPattern(type=LoadPatternType.POISSON, target_qps=target_qps), client=HTTPClientConfig( num_workers=1, warmup_connections=0, max_connections=10 @@ -121,7 +121,7 @@ def test_concurrency_benchmark( type=TestType.ONLINE, model_params=ModelParams(name="echo-server", streaming=streaming), settings=Settings( - runtime=RuntimeConfig(min_duration_ms=2000), + timeouts=Timeouts(min_duration_ms=2000), load_pattern=LoadPattern( type=LoadPatternType.CONCURRENCY, target_concurrency=4 ), @@ -192,7 +192,7 @@ def test_mode_logging(self, mock_http_echo_server, ds_dataset_path, caplog): ( TestType.OFFLINE, Settings( - runtime=RuntimeConfig(min_duration_ms=0), + timeouts=Timeouts(min_duration_ms=0), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig( num_workers=1, warmup_connections=0, max_connections=10 @@ -202,7 +202,7 @@ def test_mode_logging(self, mock_http_echo_server, ds_dataset_path, caplog): ( TestType.ONLINE, Settings( - runtime=RuntimeConfig(min_duration_ms=0), + timeouts=Timeouts(min_duration_ms=0), load_pattern=LoadPattern( type=LoadPatternType.CONCURRENCY, target_concurrency=1 ), @@ -297,7 +297,7 @@ def test_cli_run_dispatches_main_run_before_audit( model_params=ModelParams(name="echo-server", streaming=StreamingMode.OFF), datasets=[Dataset(path=ds_dataset_path, type=DatasetType.PERFORMANCE)], settings=Settings( - runtime=RuntimeConfig(min_duration_ms=0), + timeouts=Timeouts(min_duration_ms=0), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig( num_workers=1, warmup_connections=0, max_connections=10 diff --git a/tests/integration/commands/test_run_timeout.py b/tests/integration/commands/test_run_timeout.py new file mode 100644 index 000000000..1ba36b2fd --- /dev/null +++ b/tests/integration/commands/test_run_timeout.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Whole-run watchdog (settings.timeouts.run_timeout_s) integration tests. + +Locking invariant: a fired run watchdog must never produce a COMPLETE +report. The watchdog SIGTERMs the metrics aggregator (whose handler writes +an INTERRUPTED final snapshot) before stopping the session, and +``run_benchmark`` exits non-zero via ``ExecutionError``. +""" + +import json + +import pytest +from inference_endpoint.commands.benchmark.execute import run_benchmark +from inference_endpoint.config.schema import ( + BenchmarkConfig, + Dataset, + DatasetType, + EndpointConfig, + LoadPattern, + LoadPatternType, + ModelParams, + Settings, + StreamingMode, + TestMode, + TestType, + WarmupConfig, +) +from inference_endpoint.config.timeouts import Timeouts +from inference_endpoint.endpoint_client.config import HTTPClientConfig +from inference_endpoint.exceptions import ExecutionError + + +@pytest.mark.integration +def test_run_timeout_produces_interrupted_report( + mock_http_echo_server, ds_dataset_path, tmp_path +): + """run_timeout_s firing mid-run aborts with an INTERRUPTED report.""" + config = BenchmarkConfig( + type=TestType.ONLINE, + endpoint_config=EndpointConfig(endpoints=[mock_http_echo_server.url]), + model_params=ModelParams(name="echo-server", streaming=StreamingMode.OFF), + datasets=[Dataset(path=str(ds_dataset_path), type=DatasetType.PERFORMANCE)], + report_dir=tmp_path, + settings=Settings( + load_pattern=LoadPattern(type=LoadPatternType.POISSON, target_qps=5), + client=HTTPClientConfig( + num_workers=1, warmup_connections=0, max_connections=10 + ), + # Long workload so only the watchdog can end the run. + timeouts=Timeouts(min_duration_ms=120_000, run_timeout_s=2.0), + warmup=WarmupConfig(enabled=False), + ), + ) + + with pytest.raises(ExecutionError, match="Run timeout"): + run_benchmark(config, TestMode.PERF) + + snapshot_path = tmp_path / "metrics" / "final_snapshot.json" + assert snapshot_path.exists(), "aggregator must still write a final snapshot" + snapshot = json.loads(snapshot_path.read_text()) + assert snapshot["state"] == "interrupted" + + # Locking invariant: a fired run watchdog must never yield a COMPLETE report. + summary = json.loads((tmp_path / "result_summary.json").read_text()) + assert summary["complete"] is False diff --git a/tests/integration/commands/test_warmup.py b/tests/integration/commands/test_warmup.py index 4622365e5..5c52f795c 100644 --- a/tests/integration/commands/test_warmup.py +++ b/tests/integration/commands/test_warmup.py @@ -53,6 +53,7 @@ TestMode, WarmupConfig, ) +from inference_endpoint.config.timeouts import Timeouts from inference_endpoint.core.types import QueryResult, TextModelOutput from inference_endpoint.endpoint_client.config import HTTPClientConfig from inference_endpoint.openai.openai_adapter import OpenAIAdapter @@ -96,7 +97,8 @@ def _offline_config( model_params=ModelParams(name="test-model", streaming=StreamingMode.OFF), datasets=[ConfigDataset(path=str(dataset_path), type=DatasetType.PERFORMANCE)], settings=OfflineSettings( - runtime=RuntimeConfig(min_duration_ms=0, n_samples_to_issue=n_perf_samples), + runtime=RuntimeConfig(n_samples_to_issue=n_perf_samples), + timeouts=Timeouts(min_duration_ms=0), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=_MINIMAL_CLIENT, warmup=warmup, diff --git a/tests/unit/commands/test_benchmark.py b/tests/unit/commands/test_benchmark.py index 491dc72b6..afb83195a 100644 --- a/tests/unit/commands/test_benchmark.py +++ b/tests/unit/commands/test_benchmark.py @@ -53,7 +53,6 @@ from inference_endpoint.config.schema import ( BenchmarkConfig, DatasetType, - DrainConfig, LoadPattern, LoadPatternType, OfflineSettings, @@ -72,6 +71,7 @@ from inference_endpoint.config.schema import ( OnlineBenchmarkConfig as OnlineConfig, ) +from inference_endpoint.config.timeouts import Timeouts from inference_endpoint.config.utils import cli_error_formatter as _error_formatter from inference_endpoint.core.types import QueryResult from inference_endpoint.dataset_manager.dataset import Dataset @@ -125,14 +125,15 @@ def test_mode_defaults(self, cls, extra_kwargs, expected_type, expected_streamin config = cls(**_OFFLINE_KWARGS, **extra_kwargs) assert config.type == expected_type assert config.model_params.streaming == expected_streaming - assert config.settings.runtime.min_duration_ms == 600000 + assert config.settings.timeouts.min_duration_ms == 600000 @pytest.mark.unit def test_num_samples_override(self): config = OfflineConfig( **_OFFLINE_KWARGS, settings=OfflineSettings( - runtime=RuntimeConfig(min_duration_ms=0, n_samples_to_issue=100) + runtime=RuntimeConfig(n_samples_to_issue=100), + timeouts=Timeouts(min_duration_ms=0), ), ) assert config.settings.runtime.n_samples_to_issue == 100 @@ -165,9 +166,9 @@ class TestDurationSuffix: def test_duration_suffix(self, value, expected_ms): config = OfflineConfig( **_OFFLINE_KWARGS, - settings=OfflineSettings(runtime=RuntimeConfig(min_duration_ms=value)), + settings=OfflineSettings(timeouts=Timeouts(min_duration_ms=value)), ) - assert config.settings.runtime.min_duration_ms == expected_ms + assert config.settings.timeouts.min_duration_ms == expected_ms class TestDatasetParsing: @@ -341,7 +342,7 @@ def test_from_config_handler(self, mock_run, tmp_path): config_file.write_text(yaml_content) from_config(config=config_file, timeout=42.0, mode=TestMode.BOTH) called_config, called_mode = mock_run.call_args[0] - assert called_config.timeout == 42.0 + assert called_config.settings.timeouts.run_timeout_s == 42.0 assert called_mode == TestMode.BOTH @pytest.mark.unit @@ -555,41 +556,41 @@ def test_warmup_default_in_settings(self): assert warmup.n_requests is None -class TestDrainConfig: - """Tests for DrainConfig schema model.""" +class TestTimeoutsDrainFields: + """Tests for the drain-deadline fields on the Timeouts schema model.""" @pytest.mark.unit def test_defaults(self): - cfg = DrainConfig() - assert cfg.warmup_timeout_s == 240.0 - assert cfg.performance_timeout_s == 240.0 - assert cfg.accuracy_timeout_s is None - assert cfg.metrics_drain_timeout_s == 0.0 + cfg = Timeouts() + assert cfg.warmup_drain_timeout_s == 240.0 + assert cfg.performance_drain_timeout_s == 240.0 + assert cfg.accuracy_drain_timeout_s is None + assert cfg.metrics_drain_timeout_s is None @pytest.mark.unit @pytest.mark.parametrize( "field", - ["warmup_timeout_s", "performance_timeout_s", "accuracy_timeout_s"], + [ + "warmup_drain_timeout_s", + "performance_drain_timeout_s", + "accuracy_drain_timeout_s", + "metrics_drain_timeout_s", + ], ) @pytest.mark.parametrize("value", [0, -1.0]) def test_timeout_must_be_positive_or_none(self, field, value): with pytest.raises(ValidationError): - DrainConfig(**{field: value}) + Timeouts(**{field: value}) @pytest.mark.unit - def test_metrics_drain_timeout_zero_is_valid(self): - cfg = DrainConfig(metrics_drain_timeout_s=0) - assert cfg.metrics_drain_timeout_s == 0.0 - - @pytest.mark.unit - def test_metrics_drain_timeout_negative_rejected(self): - with pytest.raises(ValidationError): - DrainConfig(metrics_drain_timeout_s=-1.0) + def test_metrics_drain_timeout_none_is_unlimited(self): + cfg = Timeouts(metrics_drain_timeout_s=None) + assert cfg.metrics_drain_timeout_s is None @pytest.mark.unit def test_extra_fields_rejected(self): with pytest.raises(ValidationError): - DrainConfig(unknown_field=1) + Timeouts(unknown_field=1) @pytest.mark.unit def test_yaml_roundtrip(self, tmp_path): @@ -602,20 +603,20 @@ def test_yaml_roundtrip(self, tmp_path): datasets: - path: "test.jsonl" settings: - drain: - warmup_timeout_s: 12.5 - performance_timeout_s: 30.0 - accuracy_timeout_s: null + timeouts: + warmup_drain_timeout_s: 12.5 + performance_drain_timeout_s: 30.0 + accuracy_drain_timeout_s: null metrics_drain_timeout_s: 300.0 """ - config_file = tmp_path / "drain.yaml" + config_file = tmp_path / "timeouts.yaml" config_file.write_text(yaml_content) config = BenchmarkConfig.from_yaml_file(config_file) - drain = config.settings.drain - assert drain.warmup_timeout_s == 12.5 - assert drain.performance_timeout_s == 30.0 - assert drain.accuracy_timeout_s is None - assert drain.metrics_drain_timeout_s == 300.0 + timeouts = config.settings.timeouts + assert timeouts.warmup_drain_timeout_s == 12.5 + assert timeouts.performance_drain_timeout_s == 30.0 + assert timeouts.accuracy_drain_timeout_s is None + assert timeouts.metrics_drain_timeout_s == 300.0 class TestAggregatorArgs: @@ -653,7 +654,7 @@ def _make_ctx(self, config, tmp_path): @pytest.mark.asyncio @pytest.mark.parametrize( "timeout_s, expected_flag", - [(120.0, "120.0"), (0.0, "0.0"), (60.0, "60.0")], + [(120.0, "120.0"), (None, "0.0"), (60.0, "60.0")], ) async def test_drain_timeout_forwarded_to_aggregator_args( self, tmp_path, timeout_s, expected_flag @@ -661,7 +662,7 @@ async def test_drain_timeout_forwarded_to_aggregator_args( config = OfflineConfig( **_OFFLINE_KWARGS, settings=OfflineSettings( - drain=DrainConfig(metrics_drain_timeout_s=timeout_s) + timeouts=Timeouts(metrics_drain_timeout_s=timeout_s) ), ) ctx = self._make_ctx(config, tmp_path) @@ -711,7 +712,7 @@ async def _capture_launch(service_configs, *, timeout): async def test_tokenizer_and_workers_forwarded_from_schema(self, tmp_path): """The benchmark forwards --tokenizer and --tokenizer-workers; the workers value comes from the schema default - (drain.metrics_tokenizer_workers), the single source of truth.""" + (metrics.tokenizer_workers), the single source of truth.""" config = OfflineConfig(**_OFFLINE_KWARGS, settings=OfflineSettings()) ctx = self._make_ctx(config, tmp_path) ctx.tokenizer_name = "gpt2" @@ -755,7 +756,7 @@ async def _capture_launch(service_configs, *, timeout): idx = args.index("--tokenizer") assert args[idx + 1] == "gpt2" idx = args.index("--tokenizer-workers") - expected = str(config.settings.drain.metrics_tokenizer_workers) + expected = str(config.settings.metrics.tokenizer_workers) assert args[idx + 1] == expected @pytest.mark.unit @@ -1136,10 +1137,10 @@ def test_configured_drain_timeouts_propagate_to_phases( config = OfflineConfig( **_OFFLINE_KWARGS, settings=OfflineSettings( - drain=DrainConfig( - warmup_timeout_s=7.0, - performance_timeout_s=15.0, - accuracy_timeout_s=45.0, + timeouts=Timeouts( + warmup_drain_timeout_s=7.0, + performance_drain_timeout_s=15.0, + accuracy_drain_timeout_s=45.0, ), warmup=WarmupConfig(enabled=True, drain=True), ), diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index eb717fb13..eb4a41bb4 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -360,7 +360,7 @@ def test_negative_min_duration_rejected(self): model_params={"name": "M"}, endpoint_config={"endpoints": ["http://x"]}, datasets=[{"path": "D"}], - settings={"runtime": {"min_duration_ms": -1}}, + settings={"timeouts": {"min_duration_ms": -1}}, ) @pytest.mark.unit @@ -372,7 +372,7 @@ def test_max_lt_min_duration_rejected(self): endpoint_config={"endpoints": ["http://x"]}, datasets=[{"path": "D"}], settings={ - "runtime": {"min_duration_ms": 5000, "max_duration_ms": 1000} + "timeouts": {"min_duration_ms": 5000, "max_duration_ms": 1000} }, ) @@ -384,7 +384,7 @@ def test_max_duration_below_zero_rejected(self): model_params={"name": "M"}, endpoint_config={"endpoints": ["http://x"]}, datasets=[{"path": "D"}], - settings={"runtime": {"max_duration_ms": -1}}, + settings={"timeouts": {"max_duration_ms": -1}}, ) @pytest.mark.unit @@ -511,7 +511,7 @@ def test_max_duration_zero_converts_to_none_in_runtime_settings(self): model_params={"name": "M"}, endpoint_config={"endpoints": ["http://x"]}, datasets=[{"path": "D"}], - settings={"runtime": {"max_duration_ms": 0}}, + settings={"timeouts": {"max_duration_ms": 0}}, ) rt = RuntimeSettings.from_config(config, dataloader_num_samples=100) assert rt.max_duration_ms is None @@ -805,7 +805,7 @@ def test_agentic_inference_uses_dataset_size_ignoring_duration(self): datasets=[{"path": "D", "agentic_inference": {}}], settings={ "load_pattern": {"type": "agentic_inference", "target_concurrency": 4}, - "runtime": {"min_duration_ms": 600000}, + "timeouts": {"min_duration_ms": 600000}, }, ) rt = RuntimeSettings.from_config(config, dataloader_num_samples=4316) diff --git a/tests/unit/config/test_yaml_loader.py b/tests/unit/config/test_yaml_loader.py index 716055dd6..7bcbf105d 100644 --- a/tests/unit/config/test_yaml_loader.py +++ b/tests/unit/config/test_yaml_loader.py @@ -45,7 +45,7 @@ def test_load_valid_yaml(self, tmp_path): path: "test.jsonl" settings: - runtime: + timeouts: min_duration_ms: 60000 load_pattern: type: "max_throughput" @@ -175,7 +175,7 @@ def test_create_default_offline_config(self): config = BenchmarkConfig.create_default_config(BenchmarkTestType.OFFLINE) assert isinstance(config, BenchmarkConfig) assert config.settings.load_pattern.type == LoadPatternType.MAX_THROUGHPUT - assert config.settings.runtime.min_duration_ms == 600000 + assert config.settings.timeouts.min_duration_ms == 600000 assert config.settings.client.num_workers >= 1 # auto-resolved from -1 def test_create_default_online_config(self): @@ -183,7 +183,7 @@ def test_create_default_online_config(self): assert isinstance(config, BenchmarkConfig) assert config.settings.load_pattern.type == LoadPatternType.POISSON assert config.settings.load_pattern.target_qps == 10.0 - assert config.settings.runtime.min_duration_ms == 600000 + assert config.settings.timeouts.min_duration_ms == 600000 def test_create_default_eval_not_implemented(self): with pytest.raises(CLIError, match="EVAL"): From 35f118fa13e755dcfe19e6f0f6c94cc7c5b50094 Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Mon, 13 Jul 2026 13:31:12 -0700 Subject: [PATCH 2/5] feat(config): perf drain unbounded by default; pin whole-phase duration semantics - performance_drain_timeout_s default 240.0 -> None (wait indefinitely), matching the accuracy drain default. - min/max_duration_ms help/description now state they bound the whole performance phase (single phase; not per-dataset) and never bound warmup/accuracy. Templates regenerated. Co-Authored-By: Claude Fable 5 --- .../templates/concurrency_template.yaml | 4 ++-- .../templates/concurrency_template_full.yaml | 6 ++--- .../config/templates/offline_template.yaml | 4 ++-- .../templates/offline_template_full.yaml | 6 ++--- .../config/templates/online_template.yaml | 4 ++-- .../templates/online_template_full.yaml | 6 ++--- src/inference_endpoint/config/timeouts.py | 23 +++++++++++++++---- tests/unit/commands/test_benchmark.py | 2 +- 8 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/inference_endpoint/config/templates/concurrency_template.yaml b/src/inference_endpoint/config/templates/concurrency_template.yaml index 7a316f0af..3bc6b7b21 100644 --- a/src/inference_endpoint/config/templates/concurrency_template.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template.yaml @@ -12,8 +12,8 @@ settings: runtime: n_samples_to_issue: null # Sample count override timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) + min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) + max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) load_pattern: type: concurrency # Load pattern type | options: max_throughput, poisson, concurrency, agentic_inference, burst, step target_concurrency: 32 # Concurrent requests diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index caf741fa7..718dda8be 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -81,12 +81,12 @@ settings: min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) + min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) + max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_drain_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) + performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. metrics: diff --git a/src/inference_endpoint/config/templates/offline_template.yaml b/src/inference_endpoint/config/templates/offline_template.yaml index c1a0db1c1..27665826f 100644 --- a/src/inference_endpoint/config/templates/offline_template.yaml +++ b/src/inference_endpoint/config/templates/offline_template.yaml @@ -12,8 +12,8 @@ settings: runtime: n_samples_to_issue: null # Sample count override timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) + min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) + max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index 52530a3ca..7283c860b 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -81,12 +81,12 @@ settings: min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) + min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) + max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_drain_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) + performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. metrics: diff --git a/src/inference_endpoint/config/templates/online_template.yaml b/src/inference_endpoint/config/templates/online_template.yaml index c359e108c..0b8457c93 100644 --- a/src/inference_endpoint/config/templates/online_template.yaml +++ b/src/inference_endpoint/config/templates/online_template.yaml @@ -12,8 +12,8 @@ settings: runtime: n_samples_to_issue: null # Sample count override timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) + min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) + max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) load_pattern: type: poisson # Load pattern type | options: max_throughput, poisson, concurrency, agentic_inference, burst, step target_qps: 10.0 # Target QPS diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index e5a6af91a..26832384a 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -82,12 +82,12 @@ settings: min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Min duration (ms, or with suffix: 600s, 10m) - max_duration_ms: 0 # Maximum performance-phase duration in ms (0 for no limit) + min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) + max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) - performance_drain_timeout_s: 240.0 # Performance drain timeout in seconds (None = wait indefinitely) + performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely; accuracy is unbounded by default because every sample must complete) metrics_drain_timeout_s: null # Wall-clock budget (seconds) to finish tokenizing buffered samples after ENDED (None = wait indefinitely). An incomplete drain is surfaced via n_pending_tasks > 0, never silently dropped. metrics: diff --git a/src/inference_endpoint/config/timeouts.py b/src/inference_endpoint/config/timeouts.py index f895ef7e8..fade92fe9 100644 --- a/src/inference_endpoint/config/timeouts.py +++ b/src/inference_endpoint/config/timeouts.py @@ -19,7 +19,9 @@ - **Workload durations** (``min_duration_ms``/``max_duration_ms``): part of the benchmark definition — they shape sample-count math and bound the - performance phase only. Reaching them is a *normal* end of the run. + performance phase only, as a whole (there is a single performance phase; + they are not per-dataset knobs). Reaching them is a *normal* end of the + run. - **Give-up deadlines** (everything else): failure handling. Reaching one means something is stuck. ``run_timeout_s`` is the whole-run watchdog: when it fires the run is aborted and the report is marked INTERRUPTED — @@ -53,13 +55,24 @@ class Timeouts(WithUpdatesMixin, BaseModel): min_duration_ms: Annotated[ int, cyclopts.Parameter( - alias="--duration", help="Min duration (ms, or with suffix: 600s, 10m)" + alias="--duration", + help="Min performance-phase duration (ms, or with suffix: 600s, 10m)", ), - ] = Field(600000, ge=0) + ] = Field( + 600000, + ge=0, + description=( + "Minimum duration of the whole performance phase in ms " + "(drives sample-count math)" + ), + ) max_duration_ms: int = Field( 0, ge=0, - description="Maximum performance-phase duration in ms (0 for no limit)", + description=( + "Maximum duration of the whole performance phase in ms " + "(0 for no limit; never bounds warmup/accuracy)" + ), ) # --- Whole-run watchdog --- @@ -113,7 +126,7 @@ class Timeouts(WithUpdatesMixin, BaseModel): help="Performance drain timeout in seconds (None = wait indefinitely)", ), ] = Field( - 240.0, + None, gt=0, description="Performance drain timeout in seconds (None = wait indefinitely)", ) diff --git a/tests/unit/commands/test_benchmark.py b/tests/unit/commands/test_benchmark.py index afb83195a..002e79c84 100644 --- a/tests/unit/commands/test_benchmark.py +++ b/tests/unit/commands/test_benchmark.py @@ -563,7 +563,7 @@ class TestTimeoutsDrainFields: def test_defaults(self): cfg = Timeouts() assert cfg.warmup_drain_timeout_s == 240.0 - assert cfg.performance_drain_timeout_s == 240.0 + assert cfg.performance_drain_timeout_s is None assert cfg.accuracy_drain_timeout_s is None assert cfg.metrics_drain_timeout_s is None From bb8833b71c60cc976b2b552b9be50cd80efe1b4e Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Mon, 13 Jul 2026 13:40:51 -0700 Subject: [PATCH 3/5] refactor(config): split schema.py into focused modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit schema.py (1415 lines) now holds only BenchmarkConfig/EndpointConfig and re-exports the full schema surface; models move to sibling modules: enums.py, audit.py, model_params.py, datasets.py, settings.py (timeouts.py already existed). Import sites are unchanged — config.schema remains the single import point. Dead code deleted: SystemDefaults (DEFAULT_TIMEOUT had no consumers; DEFAULT_METRIC inlined into rulesets/mlcommons/rules.py) and the unused TEMPLATE_TYPE_MAP. regenerate-templates pre-commit hook now watches the new modules. Co-Authored-By: Claude Fable 5 --- .pre-commit-config.yaml | 2 +- AGENTS.md | 9 +- src/inference_endpoint/config/audit.py | 73 ++ src/inference_endpoint/config/datasets.py | 264 ++++++ src/inference_endpoint/config/enums.py | 131 +++ src/inference_endpoint/config/model_params.py | 167 ++++ .../config/rulesets/mlcommons/rules.py | 6 +- src/inference_endpoint/config/schema.py | 892 ++---------------- src/inference_endpoint/config/settings.py | 298 ++++++ 9 files changed, 1014 insertions(+), 828 deletions(-) create mode 100644 src/inference_endpoint/config/audit.py create mode 100644 src/inference_endpoint/config/datasets.py create mode 100644 src/inference_endpoint/config/enums.py create mode 100644 src/inference_endpoint/config/model_params.py create mode 100644 src/inference_endpoint/config/settings.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b3695b7a1..2f3801ab8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,7 +54,7 @@ repos: entry: python scripts/regenerate_templates.py language: system pass_filenames: false - files: ^(src/inference_endpoint/config/(schema\.py|templates/.*)|src/inference_endpoint/endpoint_client/config\.py|scripts/regenerate_templates\.py)$ + files: ^(src/inference_endpoint/config/((schema|enums|audit|model_params|datasets|settings|timeouts)\.py|templates/.*)|src/inference_endpoint/endpoint_client/config\.py|scripts/regenerate_templates\.py)$ - id: add-license-header name: Add license headers diff --git a/AGENTS.md b/AGENTS.md index 1b51edb62..196d229e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,7 +92,7 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint | **Dataset Manager** | `src/inference_endpoint/dataset_manager/` | Loads JSONL, HuggingFace, CSV, JSON, Parquet datasets. `Dataset` base class with `load_sample()`/`num_samples()` interface | | **Metrics Aggregator** | `src/inference_endpoint/async_utils/services/metrics_aggregator/` | Subprocess. Subscribes to events, aggregates per-sample metrics into a `MetricsRegistry` (counters + HDR-histogram series + raw values), publishes `MetricsSnapshot` over IPC PUB at a configurable cadence (`SessionState`: `INITIALIZE` → `LIVE` → `DRAINING` → {`COMPLETE` \| `INTERRUPTED`}). Final snapshot is atomically written to `final_snapshot.json` as the **primary** Report source; the terminal pub/sub frame is a TUI "run finished" signal only. | | **Report** | `src/inference_endpoint/metrics/report.py` | `Report.from_snapshot(dict)` — pure-function builder consuming the dict form (`snapshot_to_dict`). Reads `final_snapshot.json` directly via `json.loads` (no Struct decode). Plumbs `complete = (state == "complete" and n_pending_tasks == 0)`; renders an explicit warning for `INTERRUPTED` runs. | -| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema (`schema.py`), `Timeouts` (`config/timeouts.py` — all global durations/deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | +| **Config** | `src/inference_endpoint/config/`, `endpoint_client/config.py` | Pydantic-based YAML schema split into focused modules (`schema.py` = BenchmarkConfig + re-export hub; `enums.py`, `audit.py`, `model_params.py`, `datasets.py`, `settings.py`), `Timeouts` (`config/timeouts.py` — all global durations/deadlines in one frozen model at `settings.timeouts`, incl. the whole-run `run_timeout_s` watchdog), `HTTPClientConfig` (single Pydantic model for CLI/YAML/runtime), `RuntimeSettings` | | **CLI** | `src/inference_endpoint/main.py`, `commands/benchmark/cli.py` | cyclopts-based, auto-generated from `schema.py` and `HTTPClientConfig` Pydantic models. Flat shorthands via `cyclopts.Parameter(alias=...)` | | **Async Utils** | `src/inference_endpoint/async_utils/` | `LoopManager` (uvloop + eager_task_factory), ZMQ transport layer, generic `MessageCodec[T]`-parametrized pub/sub, event publisher | | **OpenAI/SGLang** | `src/inference_endpoint/openai/`, `sglang/` | Protocol adapters and response accumulators for different API formats. `openai_completions` adapter (`completions_adapter.py`) sends pre-tokenized token IDs to `/v1/completions`, bypassing the server chat template — required for gpt-oss-120b on vLLM. `sglang` adapter sends to `/generate` via `input_ids`. Both apply `Harmonize()` client-side. | @@ -239,7 +239,12 @@ src/inference_endpoint/ │ ├── metric.py # Metric types (Throughput, etc.) │ └── results_plots.py # Standardized run-artifact plots (matplotlib-guarded); CLI: scripts/plot_results.py ├── config/ -│ ├── schema.py # Single source of truth: Pydantic models + cyclopts annotations +│ ├── schema.py # BenchmarkConfig + EndpointConfig; re-export hub for the schema surface +│ ├── enums.py # Shared schema enums (TestType, LoadPatternType, StreamingMode, ...) +│ ├── audit.py # Audit config models (audit: YAML block) +│ ├── model_params.py # ModelParams, OSLDistribution, SubmissionReference +│ ├── datasets.py # Dataset, AccuracyConfig, AgenticInferenceConfig +│ ├── settings.py # Settings + Runtime/LoadPattern/Warmup/Metrics/Profiling configs │ ├── timeouts.py # Timeouts — all global durations + deadlines (settings.timeouts) │ ├── runtime_settings.py # RuntimeSettings + SampleOrderSpec dataclasses │ ├── ruleset_base.py # BenchmarkSuiteRuleset base diff --git a/src/inference_endpoint/config/audit.py b/src/inference_endpoint/config/audit.py new file mode 100644 index 000000000..37e5aa22c --- /dev/null +++ b/src/inference_endpoint/config/audit.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compliance-audit configuration (the YAML-only ``audit:`` block).""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from .enums import AuditTestId + + +class OutputCachingTestConfig(BaseModel): + """Configuration for the output-caching audit (MLPerf TEST04). + + The output-caching test runs two back-to-back phases — a reference run of + distinct samples and an audit run that repeats one fixed sample — then + checks that the audit QPS does not exceed the reference QPS by more than + ``threshold``. A large speedup indicates the SUT is caching responses. + + samples: reference-phase query count (required — an explicit count keeps + the per-phase completion check meaningful; a duration-driven phase has + no independent target to validate completion against) + audit_samples: audit-phase query count (None → equals samples) + sample_index: which dataset row is repeated (MLCommons performance_issue_same_index) + threshold: tolerance shared by both pass checks — each phase must complete + ≥ requested * (1 - threshold), and audit_qps must stay < ref_qps * (1 + threshold) + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + test: Literal[AuditTestId.OUTPUT_CACHING_TEST] + only: bool = Field( + False, + description="Run only the audit — skip the main benchmark (upstream-style standalone TEST04)", + ) + samples: int = Field(..., ge=1, description="Reference phase query count") + audit_samples: int | None = Field( + None, ge=1, description="Audit phase query count (default: equals samples)" + ) + sample_index: int = Field( + 0, ge=0, description="Dataset row index repeated in the audit phase" + ) + threshold: float = Field( + 0.10, + gt=0, + lt=1, + description=( + "Tolerance for both checks: each phase must complete " + "≥ requested * (1 - threshold), and audit_qps must stay " + "< ref_qps * (1 + threshold)" + ), + ) + + +# Single member today; becomes +# Annotated[OutputCachingTestConfig | ..., Field(discriminator="test")] +# when additional audit tests are added. +AuditConfig = OutputCachingTestConfig diff --git a/src/inference_endpoint/config/datasets.py b/src/inference_endpoint/config/datasets.py new file mode 100644 index 000000000..f3cfbea40 --- /dev/null +++ b/src/inference_endpoint/config/datasets.py @@ -0,0 +1,264 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dataset configuration models (``datasets:`` entries).""" + +from __future__ import annotations + +from pathlib import Path +from typing import Annotated, Any, Self + +import cyclopts +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from .enums import DatasetType, EvalMethod, ScorerMethod +from .model_params import _METRICS_DECOUPLED_OVERRIDE_KEYS, ModelParams + + +def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + """Recursively merge ``override`` into ``base`` and return the result. + + For overlapping keys whose values are both dicts, recurse; otherwise the + override value wins. Mutates a *copy* — callers can safely pass model_dump() + output. Used by ``Dataset.effective_generation_config`` so a sparse nested + override (e.g. ``{osl_distribution: {max: 512}}``) preserves siblings. + """ + out = dict(base) + for k, v in override.items(): + if isinstance(v, dict) and isinstance(out.get(k), dict): + out[k] = _deep_merge(out[k], v) + else: + out[k] = v + return out + + +class AgenticInferenceConfig(BaseModel): + """Agentic inference conversation configuration. + + Configuration for benchmarking conversational AI workloads with turn sequencing. + Enables testing agentic inference conversations where each turn depends on previous responses. + Presence of this block in the dataset config enables agentic inference mode. + + Attributes: + turn_timeout_s: Deadline between issuing a turn and receiving its + response. A timeout aborts that turn and all remaining client + turns of the same conversation because subsequent turns depend + on the timed-out response. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + turn_timeout_s: float = Field( + default=86400.0, + gt=0, + description=( + "Per-turn timeout in seconds. A timeout aborts that turn and all " + "remaining turns in the same conversation." + ), + ) + enable_salt: bool = Field( + False, + description=( + "Add deterministic salt markers before and after the system prompt " + "to prevent KV cache reuse across trajectories in agentic inference setting." + ), + ) + inject_tool_delay: bool = Field( + False, + description=( + "Pause for a predefined duration between turns. Duration is defined " + "in dataset." + ), + ) + num_trajectories_to_issue: int | None = Field( + default=None, + gt=0, + description=( + "Number of conversation trajectories to start. Defaults to one pass " + "over the dataset; values above the dataset size repeat trajectories " + "with unique logical conversation ids." + ), + ) + stop_issuing_on_first_user_complete: bool = Field( + False, + description=( + "When performance tracking stops because the first concurrency slot " + "has no next trajectory left to assign, also stop issuing future " + "turns. If false, replay continues outside the performance window " + "for accuracy/log coverage." + ), + ) + + +class Dataset(BaseModel): + """Dataset configuration. + + Name and type have smart defaults: name is auto-derived from path, + type defaults to PERFORMANCE. + + Accepts CLI strings via BeforeValidator on BenchmarkConfig.datasets: + ``[perf|acc:][,key=value...]`` + """ + + model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) + + name: str = Field("", description="Dataset name (auto-derived from path if empty)") + type: DatasetType = Field( + DatasetType.PERFORMANCE, description="Dataset purpose: performance or accuracy" + ) + path: Annotated[ + str | None, cyclopts.Parameter(alias="--dataset", help="Dataset file path") + ] = None + format: str | None = Field(None, description="Dataset format (auto-detected)") + samples: int | None = Field(None, gt=0, description="Number of samples to use") + eval_method: EvalMethod | None = Field( + None, description="Accuracy evaluation method" + ) + parser: dict[str, str] | None = Field( + None, description="Column remapping: {prompt: , system: }" + ) + generate_params: dict[str, Any] | None = Field( + None, description="Dataset-specific parameters passed to the generate() method" + ) + accuracy_config: AccuracyConfig | None = Field( + None, description="Accuracy evaluation settings" + ) + agentic_inference: AgenticInferenceConfig | None = Field( + None, description="Agentic inference conversation configuration" + ) + # Per-dataset generation config is a first-class capability: different + # accuracy datasets legitimately want different generation settings (e.g. + # per-dataset max OSL or top_p, as seen in DS-V4), and dataset-scoping also + # enables per-dataset dynamic OSL distributions. Only generation knobs are + # overridable — per-run/identity fields (`_METRICS_DECOUPLED_OVERRIDE_KEYS`: + # name / streaming / tokenizer_name) drive the single global tokenizer and + # MetricsAggregator, so overriding them per-dataset would desync ISL/OSL/ + # TTFT/TPOT accounting; they are rejected at validation. + # + # TODO(post-mortem): split ModelParams into a per-run ModelIdentity and a + # GenerationConfig, so the override surface is exactly the generation fields + # and identity fields cannot be named here at all. Field/method names use + # "generation_config" to keep that migration mechanical. + # + # Nested dicts (`osl_distribution`, `chat_template_kwargs`) are deep-merged + # so sparse overrides preserve sibling defaults. + generation_config_override: dict[str, Any] | None = Field( + None, + description=( + "Per-dataset overrides for the top-level model_params (sparse — " + "only the fields you want to override). Merged on top of " + "BenchmarkConfig.model_params at dataset-load time. Useful for " + "MLPerf-style runs where accuracy and performance use different " + "output budgets in the same fleet, e.g. " + "generation_config_override: {max_new_tokens: 32768, " + "temperature: 0.0}. NOTE: per-run/identity keys (`name`, " + "`streaming`, `tokenizer_name`) are rejected here — set them on " + "top-level model_params." + ), + ) + + @model_validator(mode="after") + def _auto_derive_name(self) -> Self: + """Derive name from path stem if not explicitly provided.""" + if not self.name and self.path: + object.__setattr__(self, "name", Path(self.path).stem) + return self + + @model_validator(mode="after") + def _validate_generation_config_override(self) -> Self: + """Fail fast on unknown keys and on per-run/identity keys the single + global tokenizer / MetricsAggregator would ignore. Override *values* + are validated at merge time (see ``effective_generation_config``) + because cross-field validation needs the base ``ModelParams`` from + ``BenchmarkConfig``. + """ + if self.generation_config_override: + keys = set(self.generation_config_override) + valid = set(ModelParams.model_fields) + bad = sorted(keys - valid) + if bad: + raise ValueError( + f"Dataset '{self.name}': unknown keys in " + f"generation_config_override: {bad}. " + f"Valid keys: {sorted(valid)}" + ) + decoupled = sorted(keys & _METRICS_DECOUPLED_OVERRIDE_KEYS) + if decoupled: + raise ValueError( + f"Dataset '{self.name}': generation_config_override keys " + f"{decoupled} are not honored per-dataset — the single " + "global tokenizer / metrics aggregator is launched from " + "top-level model_params, so a per-dataset value would " + "desync ISL/OSL/TTFT/TPOT accounting. Set them on " + "top-level model_params instead." + ) + return self + + def effective_generation_config(self, base: ModelParams) -> ModelParams: + """Return base merged with this dataset's generation-config overrides. + + Nested dicts are deep-merged so a sparse nested override preserves + sibling defaults (e.g. ``{osl_distribution: {max: 512}}`` keeps the + base ``type/mean/std/min``). The merged dict is re-validated through + ``ModelParams.model_validate`` so type-invalid scalar overrides (e.g. + ``temperature: 'hot'``) are rejected. Note that this only catches + scalar invalidity — a sparse nested override whose merged result + passes default-validation will not raise (callers that need stricter + nested validation should set ``base`` to an explicit instance). + """ + if not self.generation_config_override: + return base + merged = _deep_merge(base.model_dump(), self.generation_config_override) + return ModelParams.model_validate(merged) + + +class AccuracyConfig(BaseModel): + """Accuracy configuration. + + eval_method: Scorer to use (see ScorerMethod enum for options). + ground_truth: Column in the dataset containing ground truth. Defaults to "ground_truth". + extractor: Post-processor to extract answers from model output + (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor). + Optional for scorers that declare REQUIRES_EXTRACTOR = False (e.g. vbench). + num_repeats: Number of times to repeat the dataset for evaluation. Defaults to 1. + extras: Free-form keyword args forwarded to the scorer's ``__init__`` — + used for scorer-specific knobs that don't warrant a top-level field + (e.g. ``vbench_project_path``, ``subprocess_timeout_s`` for VBench). + + Example: + accuracy_config: + eval_method: "pass_at_1" + ground_truth: "answer" + extractor: "boxed_math_extractor" + num_repeats: 5 + extras: + vbench_project_path: "/path/to/accuracy" + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + eval_method: ScorerMethod | None = Field(None, description="Scorer method") + ground_truth: str | None = Field(None, description="Ground truth column name") + extractor: str | None = Field( + None, + description="Answer extractor (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor)", + ) + num_repeats: int = Field( + 1, ge=1, description="Repeat dataset N times for evaluation" + ) + extras: dict[str, Any] | None = Field( + None, + description="Free-form scorer kwargs (e.g. vbench_project_path, subprocess_timeout_s)", + ) diff --git a/src/inference_endpoint/config/enums.py b/src/inference_endpoint/config/enums.py new file mode 100644 index 000000000..c305e5cda --- /dev/null +++ b/src/inference_endpoint/config/enums.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared enums for the benchmark configuration schema.""" + +from __future__ import annotations + +from enum import Enum + + +class LoadPatternType(str, Enum): + """Load pattern types.""" + + MAX_THROUGHPUT = "max_throughput" # Offline: all queries at t=0 + POISSON = "poisson" # Online: fixed QPS with Poisson distribution + CONCURRENCY = "concurrency" # Online: fixed concurrent requests + AGENTIC_INFERENCE = ( + "agentic_inference" # Agentic inference conversations with turn sequencing + ) + BURST = "burst" # Burst pattern (TODO) + STEP = "step" # Step pattern (TODO) + + +class OSLDistributionType(str, Enum): + """Output Sequence Length distribution types.""" + + ORIGINAL = "original" # Use original distribution from dataset (default) + FIXED = "fixed" # Fixed length for all outputs + UNIFORM = "uniform" # Uniform distribution between min and max + NORMAL = "normal" # Normal/Gaussian distribution + + +class DatasetType(str, Enum): + """Dataset purpose type.""" + + PERFORMANCE = "performance" + ACCURACY = "accuracy" + + +class EvalMethod(str, Enum): + """Evaluation methods for accuracy testing.""" + + EXACT_MATCH = "exact_match" + CONTAINS = "contains" + JUDGE = "judge" + + +class ScorerMethod(str, Enum): + """Registered scorer methods for accuracy evaluation.""" + + PASS_AT_1 = "pass_at_1" + STRING_MATCH = "string_match" + ROUGE = "rouge" + CODE_BENCH = "code_bench_scorer" + SHOPIFY_CATEGORY_F1 = "shopify_category_f1" + AGENTIC_INFERENCE_INLINE = "agentic_inference_inline" + VBENCH = "vbench" + BFCL_V4 = "bfcl_v4" + LEGACY_MLPERF_DEEPSEEK_R1 = "legacy_mlperf_deepseek_r1" + + +class AuditTestId(str, Enum): + """Registered compliance audit test identifiers.""" + + # Output-caching audit — MLPerf TEST04 (duplicate-query caching detection). + OUTPUT_CACHING_TEST = "output_caching_test" + + +class TestMode(str, Enum): + """Test mode determining what to collect. + + - PERF: Performance metrics only (no response storage) + - ACC: Accuracy metrics (collect and evaluate responses) + - BOTH: Both performance and accuracy (selective collection by dataset type) + """ + + PERF = "perf" + ACC = "acc" + BOTH = "both" + + +class StreamingMode(str, Enum): + """Streaming mode for response handling. + + - AUTO: Automatically enable for online mode, disable for offline mode + - ON: Force streaming enabled (for TTFT metrics) + - OFF: Force streaming disabled + """ + + AUTO = "auto" + ON = "on" + OFF = "off" + + +class TestType(str, Enum): + """Test type for both config classification and execution mode. + + - OFFLINE: Max throughput benchmark (all queries at t=0) + - ONLINE: Sustained QPS benchmark (Poisson or concurrency-based) + - EVAL: Accuracy evaluation + - SUBMISSION: Official submission (may include both perf and accuracy) + """ + + OFFLINE = "offline" + ONLINE = "online" + EVAL = "eval" + SUBMISSION = "submission" + + +class ProfilerEngine(str, Enum): + """Inference engine whose profiling protocol the client should drive. + + Selects the HTTP path layout used to derive start/stop URLs from + ``endpoint_config.endpoints``. Each value corresponds to one server-side + profiling protocol; add a new variant + ``_PROFILE_PATHS`` row to support + another engine. + """ + + VLLM = "vllm" diff --git a/src/inference_endpoint/config/model_params.py b/src/inference_endpoint/config/model_params.py new file mode 100644 index 000000000..67be8b7c4 --- /dev/null +++ b/src/inference_endpoint/config/model_params.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Model/generation parameter models (``model_params:`` block).""" + +from __future__ import annotations + +from typing import Annotated, Any, Self + +import cyclopts +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from .enums import OSLDistributionType, StreamingMode +from .ruleset_base import BenchmarkSuiteRuleset + + +class OSLDistribution(BaseModel): + """Output Sequence Length distribution configuration. + + Distribution types: + - ORIGINAL: Use the natural distribution from the dataset (default) + - FIXED: All outputs have the same length (uses mean value) + - UNIFORM: Uniformly distributed between min and max + - NORMAL: Normal/Gaussian distribution with mean and std + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + type: OSLDistributionType = Field( + OSLDistributionType.ORIGINAL, description="Distribution type" + ) + mean: int | None = Field(None, description="Mean length (FIXED/NORMAL)") + std: int | None = Field(None, description="Std deviation (NORMAL)") + min: Annotated[ + int, + cyclopts.Parameter(alias="--min-output-tokens", help="Minimum output length"), + ] = 1 + max: int = Field(2048, description="Maximum output length") + + +class ModelParams(BaseModel): + """Model generation parameters.""" + + model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) + + name: Annotated[ + str, + cyclopts.Parameter(alias="--model", help="Model name", required=True), + ] = "" + temperature: float | None = Field(None, description="Sampling temperature") + seed: Annotated[ + int | None, + cyclopts.Parameter( + alias="--seed", help="Random seed for reproducible sampling" + ), + ] = Field(None, description="Random seed for reproducible sampling") + top_k: int | None = Field(None, description="Top-K sampling") + top_p: float | None = Field(None, description="Top-P (nucleus) sampling") + repetition_penalty: float | None = Field(None, description="Repetition penalty") + presence_penalty: float | None = Field(None, description="Presence penalty") + frequency_penalty: float | None = Field(None, description="Frequency penalty") + chat_template_kwargs: dict[str, Any] | None = Field( + None, + description="Per-request chat-template kwargs forwarded to compatible servers.", + ) + max_new_tokens: Annotated[ + int, cyclopts.Parameter(alias="--max-output-tokens", help="Max output tokens") + ] = 1024 + min_new_tokens: int = Field( + 1, + ge=0, + description="Minimum output tokens for OpenAI text-completions servers", + ) + skip_special_tokens: bool = Field( + True, + description=( + "Whether OpenAI text-completions servers omit special tokens from decoded output" + ), + ) + osl_distribution: OSLDistribution | None = Field( + None, description="Output sequence length distribution" + ) + streaming: Annotated[ + StreamingMode, + cyclopts.Parameter(alias="--streaming", help="Streaming mode: auto/on/off"), + ] = StreamingMode.AUTO + tokenizer_name: Annotated[ + str | None, + cyclopts.Parameter( + alias="--tokenizer", + help="HF repo ID or local path for the tokenizer. Overrides model name for client-side token metrics (ISL/OSL/TPOT).", + ), + ] = None + + @model_validator(mode="after") + def _validate_generation_lengths(self) -> Self: + if self.min_new_tokens > self.max_new_tokens: + raise ValueError( + "min_new_tokens must be less than or equal to max_new_tokens" + ) + return self + + +class SubmissionReference(BaseModel): + """Reference configuration for official benchmark submissions. + + Links a submission to a specific model and ruleset (competition rules). + The ruleset defines constraints like min duration, sample counts, and + performance targets that must be met for a valid submission. + + Example: + submission_ref: + model: "llama-2-70b" + ruleset: "mlperf-inference-v5.1" + """ + + model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) + + model: str # Model identifier (e.g., "llama-2-70b") + ruleset: str # Ruleset name/version (e.g., "mlperf-inference-v5.1") + + def get_ruleset_instance(self) -> BenchmarkSuiteRuleset: + """Get the actual ruleset instance from registry. + + Returns: + BenchmarkSuiteRuleset instance + + Raises: + KeyError: If ruleset not found in registry + """ + from .ruleset_registry import get_ruleset + + return get_ruleset(self.ruleset) + + +# ModelParams fields that drive the single global tokenizer / MetricsAggregator +# (launched once from top-level model_params), so a per-dataset override would +# desync ISL/OSL/TTFT/TPOT accounting without changing what is measured. Rejected +# as generation_config_override keys — they are per-run/identity, not per-dataset. +_METRICS_DECOUPLED_OVERRIDE_KEYS = frozenset({"name", "streaming", "tokenizer_name"}) + + +def _non_default_completion_controls(mp: ModelParams) -> list[str]: + """Completion-only ModelParams controls set to a non-default value. + + ``min_new_tokens``/``skip_special_tokens`` are only honored by the + ``openai_completions`` adapter; ``BenchmarkConfig`` rejects them for other + ``api_type``s. Shared by the top-level and per-dataset-override checks so + both config surfaces validate identically. + """ + checks = { + "min_new_tokens": mp.min_new_tokens != 1, + "skip_special_tokens": not mp.skip_special_tokens, + } + return [name for name, non_default in checks.items() if non_default] diff --git a/src/inference_endpoint/config/rulesets/mlcommons/rules.py b/src/inference_endpoint/config/rulesets/mlcommons/rules.py index 2625d514c..ba9ad0b07 100644 --- a/src/inference_endpoint/config/rulesets/mlcommons/rules.py +++ b/src/inference_endpoint/config/rulesets/mlcommons/rules.py @@ -27,10 +27,12 @@ from .... import metrics from ...ruleset_base import BenchmarkSuiteRuleset from ...runtime_settings import RuntimeSettings -from ...schema import SystemDefaults from ...user_config import UserConfig from . import models +# Fallback metric when a ruleset resolves no explicit target. +_DEFAULT_METRIC = metrics.Throughput(0.0) + @dataclass(frozen=True) class PerModelRuleset: @@ -214,7 +216,7 @@ def apply_user_config( return _RuntimeSettings( metric_target=metric_target if metric_target is not None - else SystemDefaults.DEFAULT_METRIC, + else _DEFAULT_METRIC, reported_metrics=reported_metrics, min_duration_ms=min_duration_ms, max_duration_ms=max_duration_ms, diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index adcfa6449..fc2c73c4e 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -24,9 +24,8 @@ import logging from collections import Counter -from enum import Enum from pathlib import Path -from typing import Annotated, Any, ClassVar, Literal, Self, Union +from typing import Annotated, Any, Literal, Self, Union import cyclopts import yaml @@ -35,840 +34,87 @@ ConfigDict, Discriminator, Field, - SerializerFunctionWrapHandler, Tag, TypeAdapter, field_validator, - model_serializer, model_validator, ) -from .. import metrics from ..core.types import APIType -from ..endpoint_client.config import HTTPClientConfig from ..exceptions import CLIError from ..utils import WithUpdatesMixin -from .ruleset_base import BenchmarkSuiteRuleset +from .audit import AuditConfig, OutputCachingTestConfig +from .datasets import AccuracyConfig, AgenticInferenceConfig, Dataset +from .enums import ( + AuditTestId, + DatasetType, + EvalMethod, + LoadPatternType, + OSLDistributionType, + ProfilerEngine, + ScorerMethod, + StreamingMode, + TestMode, + TestType, +) +from .model_params import ( + ModelParams, + OSLDistribution, + SubmissionReference, + _non_default_completion_controls, +) +from .settings import ( + LoadPattern, + MetricsConfig, + OfflineSettings, + OnlineSettings, + ProfilingConfig, + RuntimeConfig, + Settings, + WarmupConfig, +) from .timeouts import Timeouts from .utils import parse_dataset_string, resolve_env_vars -logger = logging.getLogger(__name__) - - -class SystemDefaults(BaseModel): - DEFAULT_TIMEOUT: ClassVar[float] = 300.0 - DEFAULT_METRIC: ClassVar[metrics.Metric] = metrics.Throughput(0.0) - - -def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: - """Recursively merge ``override`` into ``base`` and return the result. - - For overlapping keys whose values are both dicts, recurse; otherwise the - override value wins. Mutates a *copy* — callers can safely pass model_dump() - output. Used by ``Dataset.effective_generation_config`` so a sparse nested - override (e.g. ``{osl_distribution: {max: 512}}``) preserves siblings. - """ - out = dict(base) - for k, v in override.items(): - if isinstance(v, dict) and isinstance(out.get(k), dict): - out[k] = _deep_merge(out[k], v) - else: - out[k] = v - return out - - -# ModelParams fields that drive the single global tokenizer / MetricsAggregator -# (launched once from top-level model_params), so a per-dataset override would -# desync ISL/OSL/TTFT/TPOT accounting without changing what is measured. Rejected -# as generation_config_override keys — they are per-run/identity, not per-dataset. -_METRICS_DECOUPLED_OVERRIDE_KEYS = frozenset({"name", "streaming", "tokenizer_name"}) - - -def _non_default_completion_controls(mp: ModelParams) -> list[str]: - """Completion-only ModelParams controls set to a non-default value. - - ``min_new_tokens``/``skip_special_tokens`` are only honored by the - ``openai_completions`` adapter; ``BenchmarkConfig`` rejects them for other - ``api_type``s. Shared by the top-level and per-dataset-override checks so - both config surfaces validate identically. - """ - checks = { - "min_new_tokens": mp.min_new_tokens != 1, - "skip_special_tokens": not mp.skip_special_tokens, - } - return [name for name, non_default in checks.items() if non_default] - - -class LoadPatternType(str, Enum): - """Load pattern types.""" - - MAX_THROUGHPUT = "max_throughput" # Offline: all queries at t=0 - POISSON = "poisson" # Online: fixed QPS with Poisson distribution - CONCURRENCY = "concurrency" # Online: fixed concurrent requests - AGENTIC_INFERENCE = ( - "agentic_inference" # Agentic inference conversations with turn sequencing - ) - BURST = "burst" # Burst pattern (TODO) - STEP = "step" # Step pattern (TODO) - - -class OSLDistributionType(str, Enum): - """Output Sequence Length distribution types.""" - - ORIGINAL = "original" # Use original distribution from dataset (default) - FIXED = "fixed" # Fixed length for all outputs - UNIFORM = "uniform" # Uniform distribution between min and max - NORMAL = "normal" # Normal/Gaussian distribution - - -class DatasetType(str, Enum): - """Dataset purpose type.""" - - PERFORMANCE = "performance" - ACCURACY = "accuracy" - - -class EvalMethod(str, Enum): - """Evaluation methods for accuracy testing.""" - - EXACT_MATCH = "exact_match" - CONTAINS = "contains" - JUDGE = "judge" - - -class ScorerMethod(str, Enum): - """Registered scorer methods for accuracy evaluation.""" - - PASS_AT_1 = "pass_at_1" - STRING_MATCH = "string_match" - ROUGE = "rouge" - CODE_BENCH = "code_bench_scorer" - SHOPIFY_CATEGORY_F1 = "shopify_category_f1" - AGENTIC_INFERENCE_INLINE = "agentic_inference_inline" - VBENCH = "vbench" - BFCL_V4 = "bfcl_v4" - LEGACY_MLPERF_DEEPSEEK_R1 = "legacy_mlperf_deepseek_r1" - - -class AuditTestId(str, Enum): - """Registered compliance audit test identifiers.""" - - # Output-caching audit — MLPerf TEST04 (duplicate-query caching detection). - OUTPUT_CACHING_TEST = "output_caching_test" - - -class OutputCachingTestConfig(BaseModel): - """Configuration for the output-caching audit (MLPerf TEST04). - - The output-caching test runs two back-to-back phases — a reference run of - distinct samples and an audit run that repeats one fixed sample — then - checks that the audit QPS does not exceed the reference QPS by more than - ``threshold``. A large speedup indicates the SUT is caching responses. - - samples: reference-phase query count (required — an explicit count keeps - the per-phase completion check meaningful; a duration-driven phase has - no independent target to validate completion against) - audit_samples: audit-phase query count (None → equals samples) - sample_index: which dataset row is repeated (MLCommons performance_issue_same_index) - threshold: tolerance shared by both pass checks — each phase must complete - ≥ requested * (1 - threshold), and audit_qps must stay < ref_qps * (1 + threshold) - """ - - model_config = ConfigDict(frozen=True, extra="forbid") - - test: Literal[AuditTestId.OUTPUT_CACHING_TEST] - only: bool = Field( - False, - description="Run only the audit — skip the main benchmark (upstream-style standalone TEST04)", - ) - samples: int = Field(..., ge=1, description="Reference phase query count") - audit_samples: int | None = Field( - None, ge=1, description="Audit phase query count (default: equals samples)" - ) - sample_index: int = Field( - 0, ge=0, description="Dataset row index repeated in the audit phase" - ) - threshold: float = Field( - 0.10, - gt=0, - lt=1, - description=( - "Tolerance for both checks: each phase must complete " - "≥ requested * (1 - threshold), and audit_qps must stay " - "< ref_qps * (1 + threshold)" - ), - ) - - -# Single member today; becomes -# Annotated[OutputCachingTestConfig | ..., Field(discriminator="test")] -# when additional audit tests are added. -AuditConfig = OutputCachingTestConfig - - -class TestMode(str, Enum): - """Test mode determining what to collect. - - - PERF: Performance metrics only (no response storage) - - ACC: Accuracy metrics (collect and evaluate responses) - - BOTH: Both performance and accuracy (selective collection by dataset type) - """ - - PERF = "perf" - ACC = "acc" - BOTH = "both" - - -class StreamingMode(str, Enum): - """Streaming mode for response handling. - - - AUTO: Automatically enable for online mode, disable for offline mode - - ON: Force streaming enabled (for TTFT metrics) - - OFF: Force streaming disabled - """ - - AUTO = "auto" - ON = "on" - OFF = "off" - - -class TestType(str, Enum): - """Test type for both config classification and execution mode. - - - OFFLINE: Max throughput benchmark (all queries at t=0) - - ONLINE: Sustained QPS benchmark (Poisson or concurrency-based) - - EVAL: Accuracy evaluation - - SUBMISSION: Official submission (may include both perf and accuracy) - """ - - OFFLINE = "offline" - ONLINE = "online" - EVAL = "eval" - SUBMISSION = "submission" - - -# Mapping from template type strings to TestType enums -# Single source of truth for template type conversion -TEMPLATE_TYPE_MAP = { - "offline": TestType.OFFLINE, - "online": TestType.ONLINE, - "eval": TestType.EVAL, - "submission": TestType.SUBMISSION, -} - - -class OSLDistribution(BaseModel): - """Output Sequence Length distribution configuration. - - Distribution types: - - ORIGINAL: Use the natural distribution from the dataset (default) - - FIXED: All outputs have the same length (uses mean value) - - UNIFORM: Uniformly distributed between min and max - - NORMAL: Normal/Gaussian distribution with mean and std - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - type: OSLDistributionType = Field( - OSLDistributionType.ORIGINAL, description="Distribution type" - ) - mean: int | None = Field(None, description="Mean length (FIXED/NORMAL)") - std: int | None = Field(None, description="Std deviation (NORMAL)") - min: Annotated[ - int, - cyclopts.Parameter(alias="--min-output-tokens", help="Minimum output length"), - ] = 1 - max: int = Field(2048, description="Maximum output length") - - -class ModelParams(BaseModel): - """Model generation parameters.""" - - model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) - - name: Annotated[ - str, - cyclopts.Parameter(alias="--model", help="Model name", required=True), - ] = "" - temperature: float | None = Field(None, description="Sampling temperature") - seed: Annotated[ - int | None, - cyclopts.Parameter( - alias="--seed", help="Random seed for reproducible sampling" - ), - ] = Field(None, description="Random seed for reproducible sampling") - top_k: int | None = Field(None, description="Top-K sampling") - top_p: float | None = Field(None, description="Top-P (nucleus) sampling") - repetition_penalty: float | None = Field(None, description="Repetition penalty") - presence_penalty: float | None = Field(None, description="Presence penalty") - frequency_penalty: float | None = Field(None, description="Frequency penalty") - chat_template_kwargs: dict[str, Any] | None = Field( - None, - description="Per-request chat-template kwargs forwarded to compatible servers.", - ) - max_new_tokens: Annotated[ - int, cyclopts.Parameter(alias="--max-output-tokens", help="Max output tokens") - ] = 1024 - min_new_tokens: int = Field( - 1, - ge=0, - description="Minimum output tokens for OpenAI text-completions servers", - ) - skip_special_tokens: bool = Field( - True, - description=( - "Whether OpenAI text-completions servers omit special tokens from decoded output" - ), - ) - osl_distribution: OSLDistribution | None = Field( - None, description="Output sequence length distribution" - ) - streaming: Annotated[ - StreamingMode, - cyclopts.Parameter(alias="--streaming", help="Streaming mode: auto/on/off"), - ] = StreamingMode.AUTO - tokenizer_name: Annotated[ - str | None, - cyclopts.Parameter( - alias="--tokenizer", - help="HF repo ID or local path for the tokenizer. Overrides model name for client-side token metrics (ISL/OSL/TPOT).", - ), - ] = None - - @model_validator(mode="after") - def _validate_generation_lengths(self) -> Self: - if self.min_new_tokens > self.max_new_tokens: - raise ValueError( - "min_new_tokens must be less than or equal to max_new_tokens" - ) - return self - - -class SubmissionReference(BaseModel): - """Reference configuration for official benchmark submissions. - - Links a submission to a specific model and ruleset (competition rules). - The ruleset defines constraints like min duration, sample counts, and - performance targets that must be met for a valid submission. - - Example: - submission_ref: - model: "llama-2-70b" - ruleset: "mlperf-inference-v5.1" - """ - - model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) - - model: str # Model identifier (e.g., "llama-2-70b") - ruleset: str # Ruleset name/version (e.g., "mlperf-inference-v5.1") - - def get_ruleset_instance(self) -> BenchmarkSuiteRuleset: - """Get the actual ruleset instance from registry. - - Returns: - BenchmarkSuiteRuleset instance - - Raises: - KeyError: If ruleset not found in registry - """ - from .ruleset_registry import get_ruleset - - return get_ruleset(self.ruleset) - - -class AgenticInferenceConfig(BaseModel): - """Agentic inference conversation configuration. - - Configuration for benchmarking conversational AI workloads with turn sequencing. - Enables testing agentic inference conversations where each turn depends on previous responses. - Presence of this block in the dataset config enables agentic inference mode. - - Attributes: - turn_timeout_s: Deadline between issuing a turn and receiving its - response. A timeout aborts that turn and all remaining client - turns of the same conversation because subsequent turns depend - on the timed-out response. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - turn_timeout_s: float = Field( - default=86400.0, - gt=0, - description=( - "Per-turn timeout in seconds. A timeout aborts that turn and all " - "remaining turns in the same conversation." - ), - ) - enable_salt: bool = Field( - False, - description=( - "Add deterministic salt markers before and after the system prompt " - "to prevent KV cache reuse across trajectories in agentic inference setting." - ), - ) - inject_tool_delay: bool = Field( - False, - description=( - "Pause for a predefined duration between turns. Duration is defined " - "in dataset." - ), - ) - num_trajectories_to_issue: int | None = Field( - default=None, - gt=0, - description=( - "Number of conversation trajectories to start. Defaults to one pass " - "over the dataset; values above the dataset size repeat trajectories " - "with unique logical conversation ids." - ), - ) - stop_issuing_on_first_user_complete: bool = Field( - False, - description=( - "When performance tracking stops because the first concurrency slot " - "has no next trajectory left to assign, also stop issuing future " - "turns. If false, replay continues outside the performance window " - "for accuracy/log coverage." - ), - ) - - -class Dataset(BaseModel): - """Dataset configuration. - - Name and type have smart defaults: name is auto-derived from path, - type defaults to PERFORMANCE. - - Accepts CLI strings via BeforeValidator on BenchmarkConfig.datasets: - ``[perf|acc:][,key=value...]`` - """ - - model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True) - - name: str = Field("", description="Dataset name (auto-derived from path if empty)") - type: DatasetType = Field( - DatasetType.PERFORMANCE, description="Dataset purpose: performance or accuracy" - ) - path: Annotated[ - str | None, cyclopts.Parameter(alias="--dataset", help="Dataset file path") - ] = None - format: str | None = Field(None, description="Dataset format (auto-detected)") - samples: int | None = Field(None, gt=0, description="Number of samples to use") - eval_method: EvalMethod | None = Field( - None, description="Accuracy evaluation method" - ) - parser: dict[str, str] | None = Field( - None, description="Column remapping: {prompt: , system: }" - ) - generate_params: dict[str, Any] | None = Field( - None, description="Dataset-specific parameters passed to the generate() method" - ) - accuracy_config: AccuracyConfig | None = Field( - None, description="Accuracy evaluation settings" - ) - agentic_inference: AgenticInferenceConfig | None = Field( - None, description="Agentic inference conversation configuration" - ) - # Per-dataset generation config is a first-class capability: different - # accuracy datasets legitimately want different generation settings (e.g. - # per-dataset max OSL or top_p, as seen in DS-V4), and dataset-scoping also - # enables per-dataset dynamic OSL distributions. Only generation knobs are - # overridable — per-run/identity fields (`_METRICS_DECOUPLED_OVERRIDE_KEYS`: - # name / streaming / tokenizer_name) drive the single global tokenizer and - # MetricsAggregator, so overriding them per-dataset would desync ISL/OSL/ - # TTFT/TPOT accounting; they are rejected at validation. - # - # TODO(post-mortem): split ModelParams into a per-run ModelIdentity and a - # GenerationConfig, so the override surface is exactly the generation fields - # and identity fields cannot be named here at all. Field/method names use - # "generation_config" to keep that migration mechanical. - # - # Nested dicts (`osl_distribution`, `chat_template_kwargs`) are deep-merged - # so sparse overrides preserve sibling defaults. - generation_config_override: dict[str, Any] | None = Field( - None, - description=( - "Per-dataset overrides for the top-level model_params (sparse — " - "only the fields you want to override). Merged on top of " - "BenchmarkConfig.model_params at dataset-load time. Useful for " - "MLPerf-style runs where accuracy and performance use different " - "output budgets in the same fleet, e.g. " - "generation_config_override: {max_new_tokens: 32768, " - "temperature: 0.0}. NOTE: per-run/identity keys (`name`, " - "`streaming`, `tokenizer_name`) are rejected here — set them on " - "top-level model_params." - ), - ) - - @model_validator(mode="after") - def _auto_derive_name(self) -> Self: - """Derive name from path stem if not explicitly provided.""" - if not self.name and self.path: - object.__setattr__(self, "name", Path(self.path).stem) - return self - - @model_validator(mode="after") - def _validate_generation_config_override(self) -> Self: - """Fail fast on unknown keys and on per-run/identity keys the single - global tokenizer / MetricsAggregator would ignore. Override *values* - are validated at merge time (see ``effective_generation_config``) - because cross-field validation needs the base ``ModelParams`` from - ``BenchmarkConfig``. - """ - if self.generation_config_override: - keys = set(self.generation_config_override) - valid = set(ModelParams.model_fields) - bad = sorted(keys - valid) - if bad: - raise ValueError( - f"Dataset '{self.name}': unknown keys in " - f"generation_config_override: {bad}. " - f"Valid keys: {sorted(valid)}" - ) - decoupled = sorted(keys & _METRICS_DECOUPLED_OVERRIDE_KEYS) - if decoupled: - raise ValueError( - f"Dataset '{self.name}': generation_config_override keys " - f"{decoupled} are not honored per-dataset — the single " - "global tokenizer / metrics aggregator is launched from " - "top-level model_params, so a per-dataset value would " - "desync ISL/OSL/TTFT/TPOT accounting. Set them on " - "top-level model_params instead." - ) - return self - - def effective_generation_config(self, base: ModelParams) -> ModelParams: - """Return base merged with this dataset's generation-config overrides. - - Nested dicts are deep-merged so a sparse nested override preserves - sibling defaults (e.g. ``{osl_distribution: {max: 512}}`` keeps the - base ``type/mean/std/min``). The merged dict is re-validated through - ``ModelParams.model_validate`` so type-invalid scalar overrides (e.g. - ``temperature: 'hot'``) are rejected. Note that this only catches - scalar invalidity — a sparse nested override whose merged result - passes default-validation will not raise (callers that need stricter - nested validation should set ``base`` to an explicit instance). - """ - if not self.generation_config_override: - return base - merged = _deep_merge(base.model_dump(), self.generation_config_override) - return ModelParams.model_validate(merged) - - -class AccuracyConfig(BaseModel): - """Accuracy configuration. - - eval_method: Scorer to use (see ScorerMethod enum for options). - ground_truth: Column in the dataset containing ground truth. Defaults to "ground_truth". - extractor: Post-processor to extract answers from model output - (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor). - Optional for scorers that declare REQUIRES_EXTRACTOR = False (e.g. vbench). - num_repeats: Number of times to repeat the dataset for evaluation. Defaults to 1. - extras: Free-form keyword args forwarded to the scorer's ``__init__`` — - used for scorer-specific knobs that don't warrant a top-level field - (e.g. ``vbench_project_path``, ``subprocess_timeout_s`` for VBench). - - Example: - accuracy_config: - eval_method: "pass_at_1" - ground_truth: "answer" - extractor: "boxed_math_extractor" - num_repeats: 5 - extras: - vbench_project_path: "/path/to/accuracy" - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - eval_method: ScorerMethod | None = Field(None, description="Scorer method") - ground_truth: str | None = Field(None, description="Ground truth column name") - extractor: str | None = Field( - None, - description="Answer extractor (abcd_extractor, boxed_math_extractor, identity_extractor, python_code_extractor)", - ) - num_repeats: int = Field( - 1, ge=1, description="Repeat dataset N times for evaluation" - ) - extras: dict[str, Any] | None = Field( - None, - description="Free-form scorer kwargs (e.g. vbench_project_path, subprocess_timeout_s)", - ) - - -class RuntimeConfig(BaseModel): - """Runtime configuration. - - Sample count priority (in RuntimeSettings.total_samples_to_issue()): - 1. n_samples_to_issue (if specified) — explicit override - 2. Calculated from QPS * duration — duration-based (default: 600000ms) - 3. All dataset samples — fallback when duration is 0 - - Durations live in ``settings.timeouts`` (see ``config/timeouts.py``). - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - n_samples_to_issue: Annotated[ - int | None, - cyclopts.Parameter(alias="--num-samples", help="Sample count override"), - ] = Field(None, gt=0) - scheduler_random_seed: int = Field(42, description="Scheduler RNG seed") - dataloader_random_seed: int = Field(42, description="Dataloader RNG seed") - - -@cyclopts.Parameter(name="*") -class LoadPattern(BaseModel): - """Load pattern configuration. - - Different patterns use target_qps differently: - - max_throughput: target_qps used for calculating total queries (offline, optional with default) - - poisson: target_qps sets scheduler rate (online, required - validated) - - concurrency: issue at fixed target_concurrency (online, required - validated) - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - type: Annotated[ - LoadPatternType, - cyclopts.Parameter(name="--load-pattern", help="Load pattern type"), - ] = LoadPatternType.MAX_THROUGHPUT - target_qps: Annotated[ - float | None, cyclopts.Parameter(alias="--target-qps", help="Target QPS") - ] = Field(None, gt=0) - target_concurrency: Annotated[ - int | None, - cyclopts.Parameter(alias="--concurrency", help="Concurrent requests"), - ] = Field(None, gt=0) - - # TODO(vir): remove once the formal tail-cutting mechanism lands. - use_legacy_loadgen_qps_metrics: Annotated[ - bool, - cyclopts.Parameter( - negative="--no-use-legacy-loadgen-qps-metrics", - help=( - "Only applies to the poisson load pattern. Report QPS/TPS using " - "the legacy MLPerf LoadGen Server 'completed' definition — (completed-1)/T " - "and tokens/T, T = first issued request to completion of the " - "last-issued request (see mlcommons/inference loadgen/results.cc). " - "--no-... uses endpoints-native completed/duration. Ignored for " - "non-poisson patterns." - ), - ), - ] = True - - @model_serializer(mode="wrap") - def _serialize(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: - # use_legacy_loadgen_qps_metrics only applies to poisson; drop it from - # the serialized form (and thus YAML templates) for other patterns. - data = handler(self) - if self.type != LoadPatternType.POISSON: - data.pop("use_legacy_loadgen_qps_metrics", None) - return data - - @model_validator(mode="after") - def _validate_completeness(self) -> Self: - if self.type == LoadPatternType.POISSON and ( - self.target_qps is None or self.target_qps <= 0 - ): - raise ValueError("Poisson requires --target-qps (e.g., --target-qps 100)") - if self.type == LoadPatternType.CONCURRENCY and ( - not self.target_concurrency or self.target_concurrency <= 0 - ): - raise ValueError( - "Concurrency requires --concurrency (e.g., --concurrency 10)" - ) - if self.type == LoadPatternType.AGENTIC_INFERENCE and ( - not self.target_concurrency or self.target_concurrency <= 0 - ): - raise ValueError( - "Agentic inference requires --concurrency (e.g., --concurrency 96)" - ) - return self - - def __str__(self) -> str: - """Human-readable "type (param=value)" form for logging, e.g. - ``concurrency (target_concurrency=7)`` / ``poisson (target_qps=10.0)``. - Patterns without a driving parameter render as just the type name. - """ - if self.type in ( - LoadPatternType.CONCURRENCY, - LoadPatternType.AGENTIC_INFERENCE, - ): - return f"{self.type.value} (target_concurrency={self.target_concurrency})" - if self.type == LoadPatternType.POISSON: - return f"{self.type.value} (target_qps={self.target_qps})" - return self.type.value - - -@cyclopts.Parameter(name="*") -class WarmupConfig(BaseModel): - """Warmup phase configuration. Runs before the performance phase; results are not recorded.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - enabled: Annotated[ - bool, - cyclopts.Parameter( - alias="--warmup", help="Enable warmup phase before performance run" - ), - ] = Field(False, description="Enable warmup phase before performance run") - n_requests: Annotated[ - int | None, - cyclopts.Parameter( - alias="--warmup-requests", - help="Warmup request count (None = full dataset once)", - ), - ] = Field(None, gt=0, description="Warmup request count (None = full dataset once)") - salt: Annotated[ - bool, - cyclopts.Parameter( - alias="--warmup-salt", - help="Prepend a unique random hex salt to each warmup prompt", - ), - ] = Field( - True, description="Prepend a unique random hex salt to each warmup prompt" - ) - drain: Annotated[ - bool, - cyclopts.Parameter( - alias="--warmup-drain", - help="Drain in-flight warmup requests before starting the performance phase", - ), - ] = Field( - False, - description="Drain in-flight warmup requests before starting the performance phase", - ) - warmup_random_seed: Annotated[ - int, - cyclopts.Parameter( - alias="--warmup-seed", - help="RNG seed for warmup scheduling and sample ordering", - ), - ] = Field(42, description="RNG seed for warmup scheduling and sample ordering") - - -class MetricsConfig(BaseModel): - """Metrics-aggregator tuning knobs (non-timeout; deadlines live in Timeouts).""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - tokenizer_workers: Annotated[ - int, - cyclopts.Parameter( - alias="--metrics-tokenizer-workers", - help=( - "In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT in " - "the metrics aggregator. 0 defers all tokenization to the " - "end-of-run drain, which always uses the auto-sized sharded pool." - ), - ), - ] = Field( - 2, - ge=0, - description=( - "In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT " - "(default: 2; 0 = defer everything to the end-of-run drain)." - ), - ) - - -class ProfilerEngine(str, Enum): - """Inference engine whose profiling protocol the client should drive. - - Selects the HTTP path layout used to derive start/stop URLs from - ``endpoint_config.endpoints``. Each value corresponds to one server-side - profiling protocol; add a new variant + ``_PROFILE_PATHS`` row to support - another engine. - """ - - VLLM = "vllm" - - -@cyclopts.Parameter(name="*") -class ProfilingConfig(BaseModel): - """Client-side trigger for the server's profiler. - - When ``engine`` is set, fires POST ```` at performance-phase - begin and POST ```` at performance-phase end. URLs are derived - using the engine-specific protocol from ``urls`` when set, otherwise - from ``endpoint_config.endpoints``. - Server must be launched with profiling enabled (e.g. vLLM's - ``--profiler-config.profiler=cuda|torch``); the schedule - (``delay_iterations``, ``max_iterations``) is set there, not here. - """ - - model_config = ConfigDict(extra="forbid", frozen=True) - - engine: Annotated[ - ProfilerEngine | None, - cyclopts.Parameter( - alias="--profile", - help="Profile the named inference engine around the performance phase", - ), - ] = Field( - None, - description="Profile the named inference engine around the performance phase", - ) - urls: Annotated[ - list[str] | None, - cyclopts.Parameter( - alias="--profile-urls", - help="Override URL(s) for profiler triggers; " - "defaults to endpoint_config.endpoints", - negative="", - ), - ] = Field( - None, - description="URL(s) the profiler start/stop triggers are derived from. " - "When None, derived from endpoint_config.endpoints instead. Use when " - "the profiler admin endpoint differs from the inference endpoint.", - ) - - @field_validator("urls", mode="after") - @classmethod - def _validate_url_scheme(cls, v: list[str] | None) -> list[str] | None: - if v is None: - return v - for url in v: - if not url.startswith(("http://", "https://")): - raise ValueError( - f"Profiling endpoint URL must include scheme " - f"(http:// or https://), got: {url!r}" - ) - return v - - -@cyclopts.Parameter(name="*") -class Settings(BaseModel): - """Test settings.""" - - model_config = ConfigDict(extra="forbid", frozen=True) - - runtime: RuntimeConfig = Field(default_factory=RuntimeConfig) - load_pattern: LoadPattern = Field(default_factory=LoadPattern) - client: HTTPClientConfig = Field(default_factory=HTTPClientConfig) - timeouts: Timeouts = Field( - default_factory=Timeouts, - description="All global durations and deadlines (see config/timeouts.py)", - ) - metrics: MetricsConfig = Field(default_factory=MetricsConfig) - warmup: WarmupConfig = Field(default_factory=WarmupConfig) - profiling: ProfilingConfig = Field(default_factory=ProfilingConfig) - - -class OfflineSettings(Settings): - """Offline mode default settings.""" - - load_pattern: Annotated[LoadPattern, cyclopts.Parameter(show=False)] = Field( - default_factory=lambda: LoadPattern(type=LoadPatternType.MAX_THROUGHPUT) - ) - - -class OnlineSettings(Settings): - """Online mode default settings.""" +# Re-exported schema surface: models live in focused sibling modules +# (enums/audit/model_params/datasets/settings/timeouts); this module remains +# the single import point and owns the top-level BenchmarkConfig. +__all__ = [ + "APIType", + "AccuracyConfig", + "AgenticInferenceConfig", + "AuditConfig", + "AuditTestId", + "BenchmarkConfig", + "Dataset", + "DatasetType", + "EndpointConfig", + "EvalMethod", + "LoadPattern", + "LoadPatternType", + "MetricsConfig", + "ModelParams", + "OSLDistribution", + "OSLDistributionType", + "OfflineBenchmarkConfig", + "OfflineSettings", + "OnlineBenchmarkConfig", + "OnlineSettings", + "OutputCachingTestConfig", + "ProfilerEngine", + "ProfilingConfig", + "RuntimeConfig", + "ScorerMethod", + "Settings", + "StreamingMode", + "SubmissionReference", + "TestMode", + "TestType", + "Timeouts", + "WarmupConfig", +] - pass +logger = logging.getLogger(__name__) class EndpointConfig(BaseModel): diff --git a/src/inference_endpoint/config/settings.py b/src/inference_endpoint/config/settings.py new file mode 100644 index 000000000..b8a70e9d0 --- /dev/null +++ b/src/inference_endpoint/config/settings.py @@ -0,0 +1,298 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runtime settings models (the ``settings:`` block).""" + +from __future__ import annotations + +from typing import Annotated, Any, Self + +import cyclopts +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SerializerFunctionWrapHandler, + field_validator, + model_serializer, + model_validator, +) + +from ..endpoint_client.config import HTTPClientConfig +from .enums import LoadPatternType, ProfilerEngine +from .timeouts import Timeouts + + +class RuntimeConfig(BaseModel): + """Runtime configuration. + + Sample count priority (in RuntimeSettings.total_samples_to_issue()): + 1. n_samples_to_issue (if specified) — explicit override + 2. Calculated from QPS * duration — duration-based (default: 600000ms) + 3. All dataset samples — fallback when duration is 0 + + Durations live in ``settings.timeouts`` (see ``config/timeouts.py``). + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + n_samples_to_issue: Annotated[ + int | None, + cyclopts.Parameter(alias="--num-samples", help="Sample count override"), + ] = Field(None, gt=0) + scheduler_random_seed: int = Field(42, description="Scheduler RNG seed") + dataloader_random_seed: int = Field(42, description="Dataloader RNG seed") + + +@cyclopts.Parameter(name="*") +class LoadPattern(BaseModel): + """Load pattern configuration. + + Different patterns use target_qps differently: + - max_throughput: target_qps used for calculating total queries (offline, optional with default) + - poisson: target_qps sets scheduler rate (online, required - validated) + - concurrency: issue at fixed target_concurrency (online, required - validated) + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + type: Annotated[ + LoadPatternType, + cyclopts.Parameter(name="--load-pattern", help="Load pattern type"), + ] = LoadPatternType.MAX_THROUGHPUT + target_qps: Annotated[ + float | None, cyclopts.Parameter(alias="--target-qps", help="Target QPS") + ] = Field(None, gt=0) + target_concurrency: Annotated[ + int | None, + cyclopts.Parameter(alias="--concurrency", help="Concurrent requests"), + ] = Field(None, gt=0) + + # TODO(vir): remove once the formal tail-cutting mechanism lands. + use_legacy_loadgen_qps_metrics: Annotated[ + bool, + cyclopts.Parameter( + negative="--no-use-legacy-loadgen-qps-metrics", + help=( + "Only applies to the poisson load pattern. Report QPS/TPS using " + "the legacy MLPerf LoadGen Server 'completed' definition — (completed-1)/T " + "and tokens/T, T = first issued request to completion of the " + "last-issued request (see mlcommons/inference loadgen/results.cc). " + "--no-... uses endpoints-native completed/duration. Ignored for " + "non-poisson patterns." + ), + ), + ] = True + + @model_serializer(mode="wrap") + def _serialize(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: + # use_legacy_loadgen_qps_metrics only applies to poisson; drop it from + # the serialized form (and thus YAML templates) for other patterns. + data = handler(self) + if self.type != LoadPatternType.POISSON: + data.pop("use_legacy_loadgen_qps_metrics", None) + return data + + @model_validator(mode="after") + def _validate_completeness(self) -> Self: + if self.type == LoadPatternType.POISSON and ( + self.target_qps is None or self.target_qps <= 0 + ): + raise ValueError("Poisson requires --target-qps (e.g., --target-qps 100)") + if self.type == LoadPatternType.CONCURRENCY and ( + not self.target_concurrency or self.target_concurrency <= 0 + ): + raise ValueError( + "Concurrency requires --concurrency (e.g., --concurrency 10)" + ) + if self.type == LoadPatternType.AGENTIC_INFERENCE and ( + not self.target_concurrency or self.target_concurrency <= 0 + ): + raise ValueError( + "Agentic inference requires --concurrency (e.g., --concurrency 96)" + ) + return self + + def __str__(self) -> str: + """Human-readable "type (param=value)" form for logging, e.g. + ``concurrency (target_concurrency=7)`` / ``poisson (target_qps=10.0)``. + Patterns without a driving parameter render as just the type name. + """ + if self.type in ( + LoadPatternType.CONCURRENCY, + LoadPatternType.AGENTIC_INFERENCE, + ): + return f"{self.type.value} (target_concurrency={self.target_concurrency})" + if self.type == LoadPatternType.POISSON: + return f"{self.type.value} (target_qps={self.target_qps})" + return self.type.value + + +@cyclopts.Parameter(name="*") +class WarmupConfig(BaseModel): + """Warmup phase configuration. Runs before the performance phase; results are not recorded.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + enabled: Annotated[ + bool, + cyclopts.Parameter( + alias="--warmup", help="Enable warmup phase before performance run" + ), + ] = Field(False, description="Enable warmup phase before performance run") + n_requests: Annotated[ + int | None, + cyclopts.Parameter( + alias="--warmup-requests", + help="Warmup request count (None = full dataset once)", + ), + ] = Field(None, gt=0, description="Warmup request count (None = full dataset once)") + salt: Annotated[ + bool, + cyclopts.Parameter( + alias="--warmup-salt", + help="Prepend a unique random hex salt to each warmup prompt", + ), + ] = Field( + True, description="Prepend a unique random hex salt to each warmup prompt" + ) + drain: Annotated[ + bool, + cyclopts.Parameter( + alias="--warmup-drain", + help="Drain in-flight warmup requests before starting the performance phase", + ), + ] = Field( + False, + description="Drain in-flight warmup requests before starting the performance phase", + ) + warmup_random_seed: Annotated[ + int, + cyclopts.Parameter( + alias="--warmup-seed", + help="RNG seed for warmup scheduling and sample ordering", + ), + ] = Field(42, description="RNG seed for warmup scheduling and sample ordering") + + +class MetricsConfig(BaseModel): + """Metrics-aggregator tuning knobs (non-timeout; deadlines live in Timeouts).""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + tokenizer_workers: Annotated[ + int, + cyclopts.Parameter( + alias="--metrics-tokenizer-workers", + help=( + "In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT in " + "the metrics aggregator. 0 defers all tokenization to the " + "end-of-run drain, which always uses the auto-sized sharded pool." + ), + ), + ] = Field( + 2, + ge=0, + description=( + "In-process tokenizer threads for live (mid-run) ISL/OSL/TPOT " + "(default: 2; 0 = defer everything to the end-of-run drain)." + ), + ) + + +@cyclopts.Parameter(name="*") +class ProfilingConfig(BaseModel): + """Client-side trigger for the server's profiler. + + When ``engine`` is set, fires POST ```` at performance-phase + begin and POST ```` at performance-phase end. URLs are derived + using the engine-specific protocol from ``urls`` when set, otherwise + from ``endpoint_config.endpoints``. + Server must be launched with profiling enabled (e.g. vLLM's + ``--profiler-config.profiler=cuda|torch``); the schedule + (``delay_iterations``, ``max_iterations``) is set there, not here. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + engine: Annotated[ + ProfilerEngine | None, + cyclopts.Parameter( + alias="--profile", + help="Profile the named inference engine around the performance phase", + ), + ] = Field( + None, + description="Profile the named inference engine around the performance phase", + ) + urls: Annotated[ + list[str] | None, + cyclopts.Parameter( + alias="--profile-urls", + help="Override URL(s) for profiler triggers; " + "defaults to endpoint_config.endpoints", + negative="", + ), + ] = Field( + None, + description="URL(s) the profiler start/stop triggers are derived from. " + "When None, derived from endpoint_config.endpoints instead. Use when " + "the profiler admin endpoint differs from the inference endpoint.", + ) + + @field_validator("urls", mode="after") + @classmethod + def _validate_url_scheme(cls, v: list[str] | None) -> list[str] | None: + if v is None: + return v + for url in v: + if not url.startswith(("http://", "https://")): + raise ValueError( + f"Profiling endpoint URL must include scheme " + f"(http:// or https://), got: {url!r}" + ) + return v + + +@cyclopts.Parameter(name="*") +class Settings(BaseModel): + """Test settings.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + runtime: RuntimeConfig = Field(default_factory=RuntimeConfig) + load_pattern: LoadPattern = Field(default_factory=LoadPattern) + client: HTTPClientConfig = Field(default_factory=HTTPClientConfig) + timeouts: Timeouts = Field( + default_factory=Timeouts, + description="All global durations and deadlines (see config/timeouts.py)", + ) + metrics: MetricsConfig = Field(default_factory=MetricsConfig) + warmup: WarmupConfig = Field(default_factory=WarmupConfig) + profiling: ProfilingConfig = Field(default_factory=ProfilingConfig) + + +class OfflineSettings(Settings): + """Offline mode default settings.""" + + load_pattern: Annotated[LoadPattern, cyclopts.Parameter(show=False)] = Field( + default_factory=lambda: LoadPattern(type=LoadPatternType.MAX_THROUGHPUT) + ) + + +class OnlineSettings(Settings): + """Online mode default settings.""" + + pass From ded413f51f4adc68c803f32073fc2af152f37a85 Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Mon, 13 Jul 2026 14:22:49 -0700 Subject: [PATCH 4/5] fix(timeouts): harden run watchdog per multi-AI review of PR #409 Findings from the review council (Codex review + adversary council), all verified against the code before fixing: - Watchdog now stays armed through the unbounded metrics drain: it was cancelled right after session.run, so run_timeout_s could not bound a stuck aggregator drain (wait_for_exit(None)). Cancelled after services exit instead. - Watchdog SIGTERMs only the metrics aggregator (ServiceLauncher.terminate with module suffix, replacing terminate_all): SIGTERMing the event logger dropped its buffered events.jsonl tail; the logger flushes on the ENDED event, which session.stop() still delivers. - A timed-out run skips accuracy scoring in finalize: phases that never started KeyError in scorer init and partial phases would yield misleading subset scores. Artifacts are still salvaged. - Teardown race no longer skips finalization: if session.run raises after the watchdog fired, fall through with an empty SessionResult so result_summary.json (INTERRUPTED, complete=false) is always written; run_benchmark raises the timeout ExecutionError after finalize. - run_audit maps a watchdog fire to ExecutionError naming the timeout instead of the Ctrl-C KeyboardInterrupt path (exit 130). - MetricsConfig gets cyclopts.Parameter(name='*') matching sibling settings blocks (flat --tokenizer-workers + --metrics-tokenizer-workers). - Stale drain-key name fixed in session.py docstring. Co-Authored-By: Claude Fable 5 --- .../async_utils/services/launcher.py | 17 +++--- src/inference_endpoint/commands/audit.py | 5 ++ .../commands/benchmark/execute.py | 58 +++++++++++++------ src/inference_endpoint/config/settings.py | 1 + .../load_generator/session.py | 2 +- tests/unit/compliance/test_output_caching.py | 23 ++++++++ 6 files changed, 81 insertions(+), 25 deletions(-) diff --git a/src/inference_endpoint/async_utils/services/launcher.py b/src/inference_endpoint/async_utils/services/launcher.py index b710b8bf0..add4d34f6 100644 --- a/src/inference_endpoint/async_utils/services/launcher.py +++ b/src/inference_endpoint/async_utils/services/launcher.py @@ -69,6 +69,7 @@ class ServiceLauncher: def __init__(self, zmq_context: ManagedZMQContext) -> None: self._zmq_ctx = zmq_context self._procs: list[subprocess.Popen] = [] + self._modules: list[str] = [] @property def procs(self) -> list[subprocess.Popen]: @@ -118,6 +119,7 @@ async def launch( logger.info("Launching service: %s (id=%d)", svc.module, i) proc = subprocess.Popen(cmd) self._procs.append(proc) + self._modules.append(svc.module) await receiver.wait(timeout=timeout) logger.info("All %d services ready", len(services)) @@ -151,15 +153,16 @@ def kill_all(self) -> None: if proc.poll() is None: proc.kill() - def terminate_all(self) -> None: - """SIGTERM all managed subprocesses (graceful; see kill_all for SIGKILL). + def terminate(self, module_suffix: str) -> None: + """SIGTERM managed subprocesses whose module ends with ``module_suffix``. - The metrics aggregator installs a SIGTERM handler that writes an - INTERRUPTED final snapshot before exiting — this is the abort path - for the whole-run watchdog. + Targeted so the whole-run watchdog can abort the metrics aggregator + (whose SIGTERM handler writes an INTERRUPTED final snapshot) without + killing the event logger, which flushes its buffer on the session's + ENDED event and would lose buffered records on SIGTERM. """ - for proc in self._procs: - if proc.poll() is None: + for module, proc in zip(self._modules, self._procs, strict=True): + if module.endswith(module_suffix) and proc.poll() is None: proc.terminate() def wait_for_exit(self, timeout: float | None = 60.0) -> None: diff --git a/src/inference_endpoint/commands/audit.py b/src/inference_endpoint/commands/audit.py index 7d64df7ac..15baa727d 100644 --- a/src/inference_endpoint/commands/audit.py +++ b/src/inference_endpoint/commands/audit.py @@ -133,6 +133,11 @@ def run_audit(config: BenchmarkConfig, base_report_dir: Path) -> AuditResult: # stop, so the phase returns with an "interrupted" report. Propagate it # as KeyboardInterrupt so the CLI exits 130 (interrupted), not as a # generic ExecutionError (exit 4) indistinguishable from a phase crash. + if bench.run_timed_out: + raise ExecutionError( + f"Audit phase '{spec.label}' hit the run timeout " + "(settings.timeouts.run_timeout_s); report marked INTERRUPTED" + ) if report.state == "interrupted": raise KeyboardInterrupt(f"Audit phase '{spec.label}' interrupted") # A drain-timeout (state complete but async tasks still pending) yields diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index ef9b1126e..658d65213 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -1023,11 +1023,14 @@ def _on_run_timeout() -> None: "be marked INTERRUPTED.", run_timeout_s, ) - # SIGTERM services first: the aggregator's handler writes the - # INTERRUPTED final snapshot. publish_final is idempotent, so - # the ENDED-driven finalize after session.stop() is a no-op — - # the run can never be reported COMPLETE once this fires. - launcher.terminate_all() + # SIGTERM the aggregator first: its handler writes the + # INTERRUPTED final snapshot (publish_final is first-wins, so + # the later ENDED-driven finalize is a no-op). Even if a still- + # draining aggregator finalizes as COMPLETE first, run_benchmark + # raises on run_timed_out, so a timed-out run always fails + # loudly. Targeted (not all services): the event logger flushes + # on ENDED, which session.stop() still delivers. + launcher.terminate("metrics_aggregator") session.stop() def _on_perf_phase_timeout() -> None: @@ -1079,19 +1082,26 @@ def _on_phase_start(phase: PhaseConfig) -> None: session_completed_normally = True except Exception as e: if run_timed_out: - # The watchdog SIGTERMed the services before stopping the - # session; a teardown race can surface here as a generic - # exception. Report the timeout, not the symptom. - raise ExecutionError( - f"Run timeout ({run_timeout_s}s) reached; run aborted " - "and report marked INTERRUPTED" - ) from e - raise ExecutionError(f"Benchmark execution failed: {e}") from e + # The watchdog already aborted the run; a teardown race can + # surface here as a generic exception. Fall through with an + # empty session result so finalize still writes the + # INTERRUPTED report artifacts — run_benchmark raises the + # timeout ExecutionError after finalization. + logger.exception( + "Session error after run timeout fired " + "(continuing to finalize)" + ) + result = SessionResult( + session_id=session_id, + phase_results=[], + start_time_ns=0, + end_time_ns=0, + ) + else: + raise ExecutionError(f"Benchmark execution failed: {e}") from e finally: _timeout_done = True perf_timeout.cancel() - if run_watchdog is not None: - run_watchdog.cancel() loop.remove_signal_handler(signal.SIGINT) # Fire /stop_profile for URLs whose /start_profile succeeded. # Unifies the clean phase-end path and the abort path — @@ -1127,7 +1137,12 @@ def _on_phase_start(phase: PhaseConfig) -> None: ) publisher.close() logger.info("Waiting for services to finish processing...") + # The run watchdog stays armed through this wait: the metrics + # drain is unbounded by default, so run_timeout_s must be able + # to SIGTERM a stuck aggregator drain too. await asyncio.to_thread(launcher.wait_for_exit, None) + if run_watchdog is not None: + run_watchdog.cancel() # Source the snapshot dict for Report: # 1. Preferred: the JSON file the aggregator atomically wrote @@ -1307,9 +1322,18 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None: # Write scoring artifacts + copy event log from tmpfs to disk _write_scoring_artifacts(ctx, result, bench.tmpfs_dir) - # Accuracy scoring + # Accuracy scoring. Skipped entirely when the run watchdog fired: phases + # may never have started (scorer init KeyErrors on missing sample maps) + # and partial phases would yield misleading subset scores. accuracy_scores: dict[str, Any] = {} - for eval_cfg in ctx.eval_configs: + eval_configs = ctx.eval_configs + if bench.run_timed_out and eval_configs: + logger.warning( + "Run timeout fired — skipping accuracy scoring on partial data " + "(scoring artifacts are still salvaged for inspection)" + ) + eval_configs = [] + for eval_cfg in eval_configs: try: scorer_instance = eval_cfg.scorer( eval_cfg.dataset_name, diff --git a/src/inference_endpoint/config/settings.py b/src/inference_endpoint/config/settings.py index b8a70e9d0..2cbda7a18 100644 --- a/src/inference_endpoint/config/settings.py +++ b/src/inference_endpoint/config/settings.py @@ -187,6 +187,7 @@ class WarmupConfig(BaseModel): ] = Field(42, description="RNG seed for warmup scheduling and sample ordering") +@cyclopts.Parameter(name="*") class MetricsConfig(BaseModel): """Metrics-aggregator tuning knobs (non-timeout; deadlines live in Timeouts).""" diff --git a/src/inference_endpoint/load_generator/session.py b/src/inference_endpoint/load_generator/session.py index 0d991ab1f..e9d7d7be6 100644 --- a/src/inference_endpoint/load_generator/session.py +++ b/src/inference_endpoint/load_generator/session.py @@ -379,7 +379,7 @@ def stop_current_phase(self) -> None: Also sets the drain event: if the cap fires while the phase is already inside its ``_drain_inflight`` wait (strategy task finished), cancelling - the task is a no-op, so an unbounded (``performance_timeout_s: null``) + the task is a no-op, so an unbounded (``performance_drain_timeout_s: null``) drain would otherwise hang forever on a stuck in-flight response. """ self._current_phase_stopped = True diff --git a/tests/unit/compliance/test_output_caching.py b/tests/unit/compliance/test_output_caching.py index 0a9c7168a..44d775dbe 100644 --- a/tests/unit/compliance/test_output_caching.py +++ b/tests/unit/compliance/test_output_caching.py @@ -479,6 +479,7 @@ def test_refuses_result_on_incomplete_phase(self, tmp_path, monkeypatch): incomplete = MagicMock() incomplete.complete = False bench = MagicMock() + bench.run_timed_out = False bench.report = incomplete self._patch_phase(monkeypatch, num_samples=100, bench=bench) @@ -495,12 +496,30 @@ def test_interrupted_phase_raises_keyboard_interrupt(self, tmp_path, monkeypatch interrupted.state = "interrupted" interrupted.complete = False bench = MagicMock() + bench.run_timed_out = False bench.report = interrupted self._patch_phase(monkeypatch, num_samples=100, bench=bench) with pytest.raises(KeyboardInterrupt): run_audit(config, tmp_path) + @pytest.mark.unit + def test_run_timeout_raises_execution_error(self, tmp_path, monkeypatch): + """A whole-run watchdog (settings.timeouts.run_timeout_s) firing during + an audit phase must surface as ExecutionError naming the timeout, not + as the Ctrl-C KeyboardInterrupt path.""" + config = self._audit_config() + interrupted = MagicMock() + interrupted.state = "interrupted" + interrupted.complete = False + bench = MagicMock() + bench.run_timed_out = True + bench.report = interrupted + self._patch_phase(monkeypatch, num_samples=100, bench=bench) + + with pytest.raises(ExecutionError, match="run timeout"): + run_audit(config, tmp_path) + @pytest.mark.unit def test_keyboard_interrupt_propagates(self, tmp_path, monkeypatch): """SIGINT during a phase surfaces as KeyboardInterrupt (exit 130), not a @@ -534,6 +553,7 @@ def test_strips_accuracy_datasets_from_phase_config(self, tmp_path, monkeypatch) config.datasets = [perf_ds, acc_ds] bench = MagicMock() + bench.run_timed_out = False bench.report = None # abort after the first phase's with_updates call self._patch_phase(monkeypatch, num_samples=100, bench=bench) @@ -566,6 +586,7 @@ def test_verify_zero_qps_raises_execution_error_not_bare_valueerror( report.qps = 0.0 report.n_samples_completed = 0 bench = MagicMock() + bench.run_timed_out = False bench.report = report self._patch_phase(monkeypatch, num_samples=100, bench=bench) @@ -589,6 +610,7 @@ def test_tmpfs_dir_removed_after_phase(self, tmp_path, monkeypatch): report.qps = 1.0 report.n_samples_completed = 4 bench = MagicMock() + bench.run_timed_out = False bench.report = report tmpfs_dir = tmp_path / "tmpfs" tmpfs_dir.mkdir() @@ -648,6 +670,7 @@ def test_acc_mode_phase_keeps_accuracy_datasets(self, tmp_path, monkeypatch): "inference_endpoint.commands.audit.setup_benchmark", setup_spy ) bench = MagicMock() + bench.run_timed_out = False bench.report = None # abort right after setup, before finalize matters bench.tmpfs_dir = Path("/nonexistent-tmpfs-path-for-tests") monkeypatch.setattr( From d6ac8ac0ec90951c04cc474edd9a527071b2732c Mon Sep 17 00:00:00 2001 From: Viraat Chandra Date: Wed, 15 Jul 2026 18:52:08 -0700 Subject: [PATCH 5/5] feat(config): sample count is explicit XOR duration-derived; kill 0-sentinels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit min_duration_ms and n_samples_to_issue are now mutually exclusive (Settings validator): the sample count is either an explicit count or derived from QPS x duration — previously an explicit count silently won and the configured duration was dead weight. - min_duration_ms: int | None = None. Omitting both knobs issues the dataset once. BEHAVIOR CHANGE: a bare config (neither set) previously derived qps x 600s worth of samples; it now runs the dataset once. The 0 = 'all dataset samples' sentinel is gone (0 now rejected). - max_duration_ms: int | None = None, replacing the 0 = 'no limit' sentinel; the argv-boundary 0->None conversion in RuntimeSettings is deleted (internal RuntimeSettings stays permissive for programmatic callers passing 0). - Examples/docs/templates migrated: dropped min_duration where an explicit count is set (it was silently ignored), dropped --duration 0 usage, fixed compare_with_vllm.py double-cap (it set max_duration_ms from the same value it passed as --timeout, capping one budget via two mechanisms), corrected CLI_QUICK_REFERENCE/LOCAL_TESTING stale defaults and key locations. Co-Authored-By: Claude Fable 5 --- docs/CLI_QUICK_REFERENCE.md | 13 ++++---- docs/LOCAL_TESTING.md | 9 ++--- .../compare_with_vllm.py | 4 --- ...m_gptoss_120b_per_dataset_osl_example.yaml | 1 - .../offline_llama3_8b_cnn.yaml | 1 - .../online_llama3_8b_cnn.yaml | 1 - .../offline_qwen3_vl_235b_a22b_shopify.yaml | 2 +- .../server_qwen3_vl_235b_a22b_shopify.yaml | 2 +- examples/10_Agentic_Inference/README.md | 2 +- .../kimi_agentic_benchmark.yaml | 3 -- .../online_edge_full_run.yaml | 1 - scripts/regenerate_templates.py | 4 --- .../commands/benchmark/execute.py | 12 +++++-- .../config/runtime_settings.py | 18 +++++----- src/inference_endpoint/config/settings.py | 22 ++++++++++--- .../templates/concurrency_template.yaml | 3 -- .../templates/concurrency_template_full.yaml | 4 +-- .../config/templates/offline_template.yaml | 3 -- .../templates/offline_template_full.yaml | 4 +-- .../config/templates/online_template.yaml | 3 -- .../templates/online_template_full.yaml | 4 +-- src/inference_endpoint/config/timeouts.py | 33 +++++++++++-------- .../commands/test_accuracy_pipeline.py | 2 +- .../commands/test_benchmark_command.py | 8 ++--- tests/integration/commands/test_cli.py | 2 -- tests/integration/commands/test_warmup.py | 2 +- tests/unit/commands/test_benchmark.py | 31 ++++++++++++++--- tests/unit/config/test_schema.py | 7 ++-- tests/unit/config/test_yaml_loader.py | 4 +-- 29 files changed, 113 insertions(+), 92 deletions(-) diff --git a/docs/CLI_QUICK_REFERENCE.md b/docs/CLI_QUICK_REFERENCE.md index 5a2c545b8..f85571f08 100644 --- a/docs/CLI_QUICK_REFERENCE.md +++ b/docs/CLI_QUICK_REFERENCE.md @@ -96,8 +96,9 @@ Flag names shown as `--full.dotted.path --alias`. Both forms work. - `--model-params.max-new-tokens --max-output-tokens` - Max output tokens (default: 1024) - `--model-params.osl-distribution.min --min-output-tokens` - Min output tokens (default: 1) - `--model-params.streaming --streaming` - Streaming mode: auto/on/off (default: auto) -- `--min-duration-ms --duration` - Min duration: ms default, or with suffix (600s, 10m) (default: 600000) -- `--runtime.n-samples-to-issue --num-samples` - Explicit sample count override +- `--min-duration-ms --duration` - Min perf-phase duration: ms default, or with suffix (600s, 10m); sample count = QPS x duration +- `--runtime.n-samples-to-issue --num-samples` - Explicit sample count + - `--duration` and `--num-samples` are mutually exclusive; omit both to issue the dataset once (the default) - `--client.num-workers --workers` - HTTP workers (-1=auto, default: -1) - `--client.max-connections --max-connections` - Max TCP connections (-1=unlimited) - `--endpoint-config.api-key --api-key` - API authentication @@ -284,10 +285,10 @@ datasets: settings: runtime: - min_duration_ms: 600000 # 10 minutes - n_samples_to_issue: null # Optional: explicit sample count (null = auto-calculate) scheduler_random_seed: 42 # For Poisson/distribution sampling dataloader_random_seed: 42 # For dataset shuffling + timeouts: + min_duration_ms: 600000 # 10 minutes (or set runtime.n_samples_to_issue instead; omit both = dataset once) load_pattern: type: "max_throughput" target_qps: 10.0 @@ -325,8 +326,8 @@ Note: For submission configs, `model_params.name` is optional when `submission_r **Sample Count Control:** -- Priority: `--num-samples` > calculated (target_qps × duration) > dataset size -- Default duration: 600000ms (10 minutes) +- `--num-samples` and `--duration` are mutually exclusive: explicit count, or calculated (target_qps × duration) +- Default (neither set): issue the dataset once **Mode Requirements:** diff --git a/docs/LOCAL_TESTING.md b/docs/LOCAL_TESTING.md index b8883264e..dd1b2f8f1 100644 --- a/docs/LOCAL_TESTING.md +++ b/docs/LOCAL_TESTING.md @@ -74,8 +74,7 @@ Waiting for 5 responses... uv run inference-endpoint -v benchmark offline \ --endpoints http://localhost:8765 \ --model Qwen/Qwen3-8B \ - --dataset tests/assets/datasets/dummy_1k.jsonl \ - --duration 0 + --dataset tests/assets/datasets/dummy_1k.jsonl # Production test with custom params and report generation uv run inference-endpoint -v benchmark offline \ @@ -115,7 +114,6 @@ uv run inference-endpoint -v benchmark online \ --endpoints http://localhost:8765 \ --model Qwen/Qwen3-8B \ --dataset tests/assets/datasets/dummy_1k.jsonl \ - --duration 0 \ --load-pattern poisson \ --target-qps 100 \ --report-dir online_benchmark_report @@ -311,9 +309,8 @@ uv run inference-endpoint benchmark online \ **Sample Count Control:** -- Use `--duration 0` when you want a local test to stop after exhausting the dataset instead of running for the default timed duration -- Sample priority: `--num-samples` > dataset size (when `--duration 0`) > calculated (target_qps × duration) -- Default duration: 600000ms (10 minutes) +- By default (no `--num-samples`, no `--duration`) a run stops after issuing the dataset once +- `--num-samples` (explicit count) and `--duration` (count = target_qps × duration) are mutually exclusive **Testing & Debugging:** diff --git a/examples/03_BenchmarkComparison/compare_with_vllm.py b/examples/03_BenchmarkComparison/compare_with_vllm.py index 1f8263f23..7e3e82e38 100644 --- a/examples/03_BenchmarkComparison/compare_with_vllm.py +++ b/examples/03_BenchmarkComparison/compare_with_vllm.py @@ -179,10 +179,6 @@ def generate_ie_config( "runtime": { "n_samples_to_issue": num_requests, }, - "timeouts": { - "min_duration_ms": 0, - "max_duration_ms": timeout * 1000, - }, "load_pattern": {"type": "max_throughput"}, "client": {"num_workers": workers}, }, diff --git a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml index 04f8d227c..ea677ef0e 100644 --- a/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml +++ b/examples/04_GPTOSS120B_Example/vllm_gptoss_120b_per_dataset_osl_example.yaml @@ -68,7 +68,6 @@ settings: n_samples_to_issue: 2000 # PERF phase only; accuracy phases issue their own sample counts timeouts: - min_duration_ms: 30000 max_duration_ms: 14400000 # 4h cap on the performance phase only load_pattern: diff --git a/examples/05_Llama3.1-8B_Example/offline_llama3_8b_cnn.yaml b/examples/05_Llama3.1-8B_Example/offline_llama3_8b_cnn.yaml index 064c0bfc8..56a55599b 100644 --- a/examples/05_Llama3.1-8B_Example/offline_llama3_8b_cnn.yaml +++ b/examples/05_Llama3.1-8B_Example/offline_llama3_8b_cnn.yaml @@ -33,7 +33,6 @@ settings: n_samples_to_issue: 13368 # Number of samples to issue (for offline, this should match the dataset samples) timeouts: - min_duration_ms: 60000 # 1 minute max_duration_ms: 360000 # 6 minutes (Arbitrary here, and doesn't have counterpart in legacy loadgen) load_pattern: diff --git a/examples/05_Llama3.1-8B_Example/online_llama3_8b_cnn.yaml b/examples/05_Llama3.1-8B_Example/online_llama3_8b_cnn.yaml index 8031407dc..d67b85a54 100644 --- a/examples/05_Llama3.1-8B_Example/online_llama3_8b_cnn.yaml +++ b/examples/05_Llama3.1-8B_Example/online_llama3_8b_cnn.yaml @@ -33,7 +33,6 @@ settings: n_samples_to_issue: 13368 timeouts: - min_duration_ms: 600000 # 10 minutes max_duration_ms: 3600000 # 60 minutes (Arbitrary here, and doesn't have counterpart in legacy loadgen) load_pattern: diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml index c44f715b2..eca4517ac 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/offline_qwen3_vl_235b_a22b_shopify.yaml @@ -26,7 +26,7 @@ settings: dataloader_random_seed: 42 # For dataset shuffling timeouts: - min_duration_ms: 600000 # 10 minutes, this is override when n_samples_to_issue is set + min_duration_ms: 600000 # 10 minutes (mutually exclusive with runtime.n_samples_to_issue) warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) performance_drain_timeout_s: null # Performance drain timeout in seconds (None = wait indefinitely) accuracy_drain_timeout_s: null # Accuracy drain timeout in seconds (None = wait indefinitely) diff --git a/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml b/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml index 2c3a653c4..9285d3b1a 100644 --- a/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml +++ b/examples/08_Qwen3-VL-235B-A22B_Example/server_qwen3_vl_235b_a22b_shopify.yaml @@ -26,7 +26,7 @@ settings: dataloader_random_seed: 42 timeouts: - min_duration_ms: 600000 # 10 minutes, this is override when n_samples_to_issue is set + min_duration_ms: 600000 # 10 minutes (mutually exclusive with runtime.n_samples_to_issue) load_pattern: type: "poisson" diff --git a/examples/10_Agentic_Inference/README.md b/examples/10_Agentic_Inference/README.md index a568bb2f4..4347b9b56 100644 --- a/examples/10_Agentic_Inference/README.md +++ b/examples/10_Agentic_Inference/README.md @@ -129,7 +129,7 @@ For official Kimi agentic benchmark runs, keep these values fixed: - `model_params.chat_template_kwargs.preserve_thinking: true` - First dataset `type: performance` - First dataset `accuracy_config.eval_method: agentic_inference_inline` -- `settings.timeouts.min_duration_ms: 0` +- `settings.timeouts.min_duration_ms` omitted (no duration target) - `settings.load_pattern.type: agentic_inference` - `settings.client.warmup_connections: 0` - `settings.client.max_idle_time: 0.5` diff --git a/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml b/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml index ad4318bdf..9f2a5f3b0 100644 --- a/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml +++ b/examples/10_Agentic_Inference/kimi_agentic_benchmark.yaml @@ -25,9 +25,6 @@ datasets: stop_issuing_on_first_user_complete: false settings: - timeouts: - min_duration_ms: 0 - load_pattern: type: agentic_inference target_concurrency: 8 # Submission-specific concurrency. diff --git a/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml b/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml index f20daf8a9..f5a995172 100644 --- a/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml +++ b/examples/11_Edge_Agentic_Example/online_edge_full_run.yaml @@ -89,7 +89,6 @@ settings: runtime: dataloader_random_seed: 42 # BFCL accuracy sampling seed; compliance requires 42. timeouts: - min_duration_ms: 0 # Safety cap (4 h) so the performance phase stays bounded even if decode is # slower than expected; one pass should finish in ~2.5 h on an edge box. max_duration_ms: 14400000 diff --git a/scripts/regenerate_templates.py b/scripts/regenerate_templates.py index 87e2aec40..990bb39d1 100644 --- a/scripts/regenerate_templates.py +++ b/scripts/regenerate_templates.py @@ -359,10 +359,6 @@ def _build_minimal(test_type: TestType, overrides: dict) -> dict: "runtime": { "n_samples_to_issue": None, }, - "timeouts": { - "min_duration_ms": 600000, - "max_duration_ms": 0, - }, }, "endpoint_config": {"endpoints": [PLACEHOLDER_ENDPOINT]}, } diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index 658d65213..c4bb79606 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -495,7 +495,13 @@ def setup_benchmark( ) if rt_settings is not None: logger.info( - f"Min Duration: {rt_settings.min_duration_ms / 1000:.1f}s, Expected samples: {total_samples}" + "Min Duration: %s, Expected samples: %d" + % ( + f"{rt_settings.min_duration_ms / 1000:.1f}s" + if rt_settings.min_duration_ms + else "none (dataset-once)", + total_samples, + ) ) else: logger.info(f"Accuracy-only mode, Expected samples: {total_samples}") @@ -534,7 +540,7 @@ def _build_phases( ) warmup_rt = dataclass_replace( ctx.rt_settings, - min_duration_ms=0, + min_duration_ms=None, max_duration_ms=None, n_samples_from_dataset=ctx.dataloader.num_samples(), n_samples_to_issue=warmup_cfg.n_requests, @@ -598,7 +604,7 @@ def _build_phases( acc_settings = RuntimeSettings( metric_target=rng_settings.metric_target, reported_metrics=rng_settings.reported_metrics, - min_duration_ms=0, + min_duration_ms=None, max_duration_ms=None, n_samples_from_dataset=acc_ds.num_samples(), n_samples_to_issue=acc_ds.num_samples() * acc_ds.repeats, diff --git a/src/inference_endpoint/config/runtime_settings.py b/src/inference_endpoint/config/runtime_settings.py index c51d51bfc..d7001536f 100644 --- a/src/inference_endpoint/config/runtime_settings.py +++ b/src/inference_endpoint/config/runtime_settings.py @@ -94,8 +94,9 @@ class RuntimeSettings: reported_metrics: list[metrics.Metric] """List of metrics to collect and report""" - min_duration_ms: int - """Minimum benchmark duration in milliseconds""" + min_duration_ms: int | None + """Minimum performance-phase duration in ms (None/0 = no duration target: + issue the dataset once).""" max_duration_ms: int | None """Maximum benchmark duration in milliseconds (timeout). None means no wall-clock limit.""" @@ -190,9 +191,7 @@ def _from_config_default( "metric_target": metrics.Throughput(effective_qps), "reported_metrics": [metrics.Throughput(effective_qps)], "min_duration_ms": timeouts_cfg.min_duration_ms, - "max_duration_ms": None - if timeouts_cfg.max_duration_ms == 0 - else timeouts_cfg.max_duration_ms, + "max_duration_ms": timeouts_cfg.max_duration_ms, "n_samples_from_dataset": dataloader_num_samples, "n_samples_to_issue": runtime_cfg.n_samples_to_issue, # From config (CLI --num-samples or YAML) "min_sample_count": 1, @@ -214,7 +213,7 @@ def total_samples_to_issue( Priority: 1. If `n_samples_to_issue` is set, return it (explicit override) - 2. If min_duration_ms=0, return all dataset samples (new CLI default) + 2. If no duration target is set, return all dataset samples 3. Otherwise, calculate from metric target * duration Args: @@ -252,11 +251,12 @@ def total_samples_to_issue( ) return self.n_samples_from_dataset - # If min_duration is 0, use all dataset samples (new CLI default behavior) - if self.min_duration_ms == 0: + # No duration target (None from schema, 0 from programmatic callers): + # issue the dataset once. + if not self.min_duration_ms: result = max(self.min_sample_count, self.n_samples_from_dataset) logger.debug( - f"Sample count: {result} (using all dataset samples, duration=0)" + f"Sample count: {result} (using all dataset samples, no duration target)" ) return result diff --git a/src/inference_endpoint/config/settings.py b/src/inference_endpoint/config/settings.py index 2cbda7a18..715ab3938 100644 --- a/src/inference_endpoint/config/settings.py +++ b/src/inference_endpoint/config/settings.py @@ -38,10 +38,10 @@ class RuntimeConfig(BaseModel): """Runtime configuration. - Sample count priority (in RuntimeSettings.total_samples_to_issue()): - 1. n_samples_to_issue (if specified) — explicit override - 2. Calculated from QPS * duration — duration-based (default: 600000ms) - 3. All dataset samples — fallback when duration is 0 + Sample count (mutually exclusive knobs; see Settings validator): + - n_samples_to_issue — explicit count + - timeouts.min_duration_ms — duration-derived (QPS * duration) + - neither — issue the dataset once Durations live in ``settings.timeouts`` (see ``config/timeouts.py``). """ @@ -284,6 +284,20 @@ class Settings(BaseModel): warmup: WarmupConfig = Field(default_factory=WarmupConfig) profiling: ProfilingConfig = Field(default_factory=ProfilingConfig) + @model_validator(mode="after") + def _validate_count_or_duration(self) -> Self: + if ( + self.runtime.n_samples_to_issue is not None + and self.timeouts.min_duration_ms is not None + ): + raise ValueError( + "runtime.n_samples_to_issue (--num-samples) and " + "timeouts.min_duration_ms (--duration) are mutually exclusive: " + "the sample count is either explicit or duration-derived. " + "Omit both to issue the dataset once." + ) + return self + class OfflineSettings(Settings): """Offline mode default settings.""" diff --git a/src/inference_endpoint/config/templates/concurrency_template.yaml b/src/inference_endpoint/config/templates/concurrency_template.yaml index 3bc6b7b21..f3bc05e26 100644 --- a/src/inference_endpoint/config/templates/concurrency_template.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template.yaml @@ -11,9 +11,6 @@ datasets: # Dataset configs settings: runtime: n_samples_to_issue: null # Sample count override - timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) - max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) load_pattern: type: concurrency # Load pattern type | options: max_throughput, poisson, concurrency, agentic_inference, burst, step target_concurrency: 32 # Concurrent requests diff --git a/src/inference_endpoint/config/templates/concurrency_template_full.yaml b/src/inference_endpoint/config/templates/concurrency_template_full.yaml index 718dda8be..81136f218 100644 --- a/src/inference_endpoint/config/templates/concurrency_template_full.yaml +++ b/src/inference_endpoint/config/templates/concurrency_template_full.yaml @@ -81,8 +81,8 @@ settings: min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) - max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) + min_duration_ms: null # Minimum duration of the whole performance phase in ms; the sample count is derived as QPS x duration. Mutually exclusive with runtime.n_samples_to_issue — omit both to issue the dataset once. + max_duration_ms: null # Maximum duration of the whole performance phase in ms (None for no limit; never bounds warmup/accuracy) run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) diff --git a/src/inference_endpoint/config/templates/offline_template.yaml b/src/inference_endpoint/config/templates/offline_template.yaml index 27665826f..a8b90ec53 100644 --- a/src/inference_endpoint/config/templates/offline_template.yaml +++ b/src/inference_endpoint/config/templates/offline_template.yaml @@ -11,9 +11,6 @@ datasets: # Dataset configs settings: runtime: n_samples_to_issue: null # Sample count override - timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) - max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) endpoint_config: endpoints: # Endpoint URL(s). Must include scheme, e.g. 'http://host:port'. - http://localhost:8000 diff --git a/src/inference_endpoint/config/templates/offline_template_full.yaml b/src/inference_endpoint/config/templates/offline_template_full.yaml index 7283c860b..411b707ae 100644 --- a/src/inference_endpoint/config/templates/offline_template_full.yaml +++ b/src/inference_endpoint/config/templates/offline_template_full.yaml @@ -81,8 +81,8 @@ settings: min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) - max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) + min_duration_ms: null # Minimum duration of the whole performance phase in ms; the sample count is derived as QPS x duration. Mutually exclusive with runtime.n_samples_to_issue — omit both to issue the dataset once. + max_duration_ms: null # Maximum duration of the whole performance phase in ms (None for no limit; never bounds warmup/accuracy) run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) diff --git a/src/inference_endpoint/config/templates/online_template.yaml b/src/inference_endpoint/config/templates/online_template.yaml index 0b8457c93..7404a28cc 100644 --- a/src/inference_endpoint/config/templates/online_template.yaml +++ b/src/inference_endpoint/config/templates/online_template.yaml @@ -11,9 +11,6 @@ datasets: # Dataset configs settings: runtime: n_samples_to_issue: null # Sample count override - timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) - max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) load_pattern: type: poisson # Load pattern type | options: max_throughput, poisson, concurrency, agentic_inference, burst, step target_qps: 10.0 # Target QPS diff --git a/src/inference_endpoint/config/templates/online_template_full.yaml b/src/inference_endpoint/config/templates/online_template_full.yaml index 26832384a..d99622b66 100644 --- a/src/inference_endpoint/config/templates/online_template_full.yaml +++ b/src/inference_endpoint/config/templates/online_template_full.yaml @@ -82,8 +82,8 @@ settings: min_required_connections: -1 # Min connections to initialize (-1=auto, 0=disabled) worker_gc_mode: relaxed # Worker GC strategy | options: disabled, relaxed, system timeouts: # All global durations and deadlines (see config/timeouts.py) - min_duration_ms: 600000 # Minimum duration of the whole performance phase in ms (drives sample-count math) - max_duration_ms: 0 # Maximum duration of the whole performance phase in ms (0 for no limit; never bounds warmup/accuracy) + min_duration_ms: null # Minimum duration of the whole performance phase in ms; the sample count is derived as QPS x duration. Mutually exclusive with runtime.n_samples_to_issue — omit both to issue the dataset once. + max_duration_ms: null # Maximum duration of the whole performance phase in ms (None for no limit; never bounds warmup/accuracy) run_timeout_s: null # Whole-run watchdog in seconds (None = off). Covers every phase including drains; firing aborts the run, marks the report INTERRUPTED, and exits non-zero. Never derives per-stage deadlines. service_ready_timeout_s: 30.0 # Seconds to wait for metrics-aggregator/event-logger services to become ready. warmup_drain_timeout_s: 240.0 # Warmup drain timeout in seconds (None = wait indefinitely) diff --git a/src/inference_endpoint/config/timeouts.py b/src/inference_endpoint/config/timeouts.py index fade92fe9..4cc7485c2 100644 --- a/src/inference_endpoint/config/timeouts.py +++ b/src/inference_endpoint/config/timeouts.py @@ -27,9 +27,11 @@ when it fires the run is aborted and the report is marked INTERRUPTED — it never derives or caps the per-stage deadlines below it. -``None`` means "wait indefinitely" for every optional deadline. Deadlines -are resolved to plain floats before the run starts; nothing in the hot -path reads this model. +``None`` means "wait indefinitely" for every optional deadline. For the +workload durations ``None`` means "no duration target": the sample count is +then explicit (``runtime.n_samples_to_issue``) or the dataset issued once. +Everything is resolved to plain values before the run starts; nothing in +the hot path reads this model. Dataset-scoped time knobs (e.g. agentic ``turn_timeout_s``) stay in their dataset config blocks — they are per-workload behavior, not global. @@ -53,25 +55,26 @@ class Timeouts(WithUpdatesMixin, BaseModel): # --- Workload durations (benchmark definition, not failure handling) --- min_duration_ms: Annotated[ - int, + int | None, cyclopts.Parameter( alias="--duration", help="Min performance-phase duration (ms, or with suffix: 600s, 10m)", ), ] = Field( - 600000, - ge=0, + None, + gt=0, description=( - "Minimum duration of the whole performance phase in ms " - "(drives sample-count math)" + "Minimum duration of the whole performance phase in ms; the sample " + "count is derived as QPS x duration. Mutually exclusive with " + "runtime.n_samples_to_issue — omit both to issue the dataset once." ), ) - max_duration_ms: int = Field( - 0, - ge=0, + max_duration_ms: int | None = Field( + None, + gt=0, description=( "Maximum duration of the whole performance phase in ms " - "(0 for no limit; never bounds warmup/accuracy)" + "(None for no limit; never bounds warmup/accuracy)" ), ) @@ -180,7 +183,11 @@ def _parse_duration_suffix(cls, v: object) -> object: @model_validator(mode="after") def _validate_durations(self) -> Self: - if self.max_duration_ms != 0 and self.max_duration_ms < self.min_duration_ms: + if ( + self.max_duration_ms is not None + and self.min_duration_ms is not None + and self.max_duration_ms < self.min_duration_ms + ): raise ValueError( f"max_duration_ms ({self.max_duration_ms}) must be >= " f"min_duration_ms ({self.min_duration_ms})" diff --git a/tests/integration/commands/test_accuracy_pipeline.py b/tests/integration/commands/test_accuracy_pipeline.py index d0eb3ec73..578b2cc8e 100644 --- a/tests/integration/commands/test_accuracy_pipeline.py +++ b/tests/integration/commands/test_accuracy_pipeline.py @@ -119,7 +119,7 @@ def test_accuracy_scoring_with_echo_server( ), ], settings=Settings( - timeouts=Timeouts(min_duration_ms=0), + timeouts=Timeouts(), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig( num_workers=1, warmup_connections=0, max_connections=10 diff --git a/tests/integration/commands/test_benchmark_command.py b/tests/integration/commands/test_benchmark_command.py index 04d7c9fc8..cd6728c10 100644 --- a/tests/integration/commands/test_benchmark_command.py +++ b/tests/integration/commands/test_benchmark_command.py @@ -42,7 +42,7 @@ from inference_endpoint.endpoint_client.config import HTTPClientConfig _TEST_SETTINGS = Settings( - timeouts=Timeouts(min_duration_ms=0), + timeouts=Timeouts(), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig(num_workers=1, warmup_connections=0, max_connections=10), ) @@ -192,7 +192,7 @@ def test_mode_logging(self, mock_http_echo_server, ds_dataset_path, caplog): ( TestType.OFFLINE, Settings( - timeouts=Timeouts(min_duration_ms=0), + timeouts=Timeouts(), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig( num_workers=1, warmup_connections=0, max_connections=10 @@ -202,7 +202,7 @@ def test_mode_logging(self, mock_http_echo_server, ds_dataset_path, caplog): ( TestType.ONLINE, Settings( - timeouts=Timeouts(min_duration_ms=0), + timeouts=Timeouts(), load_pattern=LoadPattern( type=LoadPatternType.CONCURRENCY, target_concurrency=1 ), @@ -297,7 +297,7 @@ def test_cli_run_dispatches_main_run_before_audit( model_params=ModelParams(name="echo-server", streaming=StreamingMode.OFF), datasets=[Dataset(path=ds_dataset_path, type=DatasetType.PERFORMANCE)], settings=Settings( - timeouts=Timeouts(min_duration_ms=0), + timeouts=Timeouts(), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=HTTPClientConfig( num_workers=1, warmup_connections=0, max_connections=10 diff --git a/tests/integration/commands/test_cli.py b/tests/integration/commands/test_cli.py index 2696b25d0..890cf96aa 100644 --- a/tests/integration/commands/test_cli.py +++ b/tests/integration/commands/test_cli.py @@ -246,8 +246,6 @@ def test_offline(self, mock_http_echo_server, ds_dataset_path, tmp_path): tmp_path, "benchmark", "offline", - "--duration", - "0", "--streaming", "off", ) diff --git a/tests/integration/commands/test_warmup.py b/tests/integration/commands/test_warmup.py index 5c52f795c..9d7efc75c 100644 --- a/tests/integration/commands/test_warmup.py +++ b/tests/integration/commands/test_warmup.py @@ -98,7 +98,7 @@ def _offline_config( datasets=[ConfigDataset(path=str(dataset_path), type=DatasetType.PERFORMANCE)], settings=OfflineSettings( runtime=RuntimeConfig(n_samples_to_issue=n_perf_samples), - timeouts=Timeouts(min_duration_ms=0), + timeouts=Timeouts(), load_pattern=LoadPattern(type=LoadPatternType.MAX_THROUGHPUT), client=_MINIMAL_CLIENT, warmup=warmup, diff --git a/tests/unit/commands/test_benchmark.py b/tests/unit/commands/test_benchmark.py index 002e79c84..ecd84374d 100644 --- a/tests/unit/commands/test_benchmark.py +++ b/tests/unit/commands/test_benchmark.py @@ -125,7 +125,7 @@ def test_mode_defaults(self, cls, extra_kwargs, expected_type, expected_streamin config = cls(**_OFFLINE_KWARGS, **extra_kwargs) assert config.type == expected_type assert config.model_params.streaming == expected_streaming - assert config.settings.timeouts.min_duration_ms == 600000 + assert config.settings.timeouts.min_duration_ms is None @pytest.mark.unit def test_num_samples_override(self): @@ -133,7 +133,7 @@ def test_num_samples_override(self): **_OFFLINE_KWARGS, settings=OfflineSettings( runtime=RuntimeConfig(n_samples_to_issue=100), - timeouts=Timeouts(min_duration_ms=0), + timeouts=Timeouts(), ), ) assert config.settings.runtime.n_samples_to_issue == 100 @@ -556,6 +556,27 @@ def test_warmup_default_in_settings(self): assert warmup.n_requests is None +class TestSampleCountKnobs: + """n_samples_to_issue and min_duration_ms are mutually exclusive.""" + + @pytest.mark.unit + def test_count_and_duration_mutually_exclusive(self): + with pytest.raises(ValueError, match="mutually exclusive"): + OfflineSettings( + runtime=RuntimeConfig(n_samples_to_issue=100), + timeouts=Timeouts(min_duration_ms=1000), + ) + + @pytest.mark.unit + def test_neither_set_issues_dataset_once(self): + from inference_endpoint.config.runtime_settings import RuntimeSettings + + config = OfflineConfig(**_OFFLINE_KWARGS) + rt = RuntimeSettings.from_config(config, dataloader_num_samples=123) + assert rt.min_duration_ms is None + assert rt.total_samples_to_issue(align_to_dataset_size=False) == 123 + + class TestTimeoutsDrainFields: """Tests for the drain-deadline fields on the Timeouts schema model.""" @@ -982,7 +1003,9 @@ def test_warmup_phase_uses_max_throughput(self, base_rt_settings, simple_dataset assert warmup_rt.load_pattern.type == LoadPatternType.MAX_THROUGHPUT @pytest.mark.unit - def test_warmup_phase_min_duration_is_zero(self, base_rt_settings, simple_dataset): + def test_warmup_phase_has_no_duration_target( + self, base_rt_settings, simple_dataset + ): config = OfflineConfig( **_OFFLINE_KWARGS, settings=OfflineSettings(warmup=WarmupConfig(enabled=True)), @@ -990,7 +1013,7 @@ def test_warmup_phase_min_duration_is_zero(self, base_rt_settings, simple_datase ctx = self._make_ctx(config, base_rt_settings, simple_dataset) phases = _build_phases(ctx) - assert phases[0].runtime_settings.min_duration_ms == 0 + assert phases[0].runtime_settings.min_duration_ms is None @pytest.mark.unit def test_warmup_phase_no_max_duration(self, base_rt_settings, simple_dataset): diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index eb4a41bb4..b029faabb 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -354,7 +354,7 @@ def test_online_max_throughput_rejected(self): @pytest.mark.unit def test_negative_min_duration_rejected(self): - with pytest.raises(ValueError, match="greater than or equal to 0"): + with pytest.raises(ValueError, match="greater than 0"): BenchmarkConfig( type=TestType.OFFLINE, model_params={"name": "M"}, @@ -378,7 +378,7 @@ def test_max_lt_min_duration_rejected(self): @pytest.mark.unit def test_max_duration_below_zero_rejected(self): - with pytest.raises(ValueError, match="greater than or equal to 0"): + with pytest.raises(ValueError, match="greater than 0"): BenchmarkConfig( type=TestType.OFFLINE, model_params={"name": "M"}, @@ -503,7 +503,7 @@ def test_to_yaml_file(self, tmp_path): assert loaded.model_params.name == "M" @pytest.mark.unit - def test_max_duration_zero_converts_to_none_in_runtime_settings(self): + def test_max_duration_defaults_to_none_in_runtime_settings(self): from inference_endpoint.config.runtime_settings import RuntimeSettings config = BenchmarkConfig( @@ -511,7 +511,6 @@ def test_max_duration_zero_converts_to_none_in_runtime_settings(self): model_params={"name": "M"}, endpoint_config={"endpoints": ["http://x"]}, datasets=[{"path": "D"}], - settings={"timeouts": {"max_duration_ms": 0}}, ) rt = RuntimeSettings.from_config(config, dataloader_num_samples=100) assert rt.max_duration_ms is None diff --git a/tests/unit/config/test_yaml_loader.py b/tests/unit/config/test_yaml_loader.py index 7bcbf105d..2cd868a85 100644 --- a/tests/unit/config/test_yaml_loader.py +++ b/tests/unit/config/test_yaml_loader.py @@ -175,7 +175,7 @@ def test_create_default_offline_config(self): config = BenchmarkConfig.create_default_config(BenchmarkTestType.OFFLINE) assert isinstance(config, BenchmarkConfig) assert config.settings.load_pattern.type == LoadPatternType.MAX_THROUGHPUT - assert config.settings.timeouts.min_duration_ms == 600000 + assert config.settings.timeouts.min_duration_ms is None assert config.settings.client.num_workers >= 1 # auto-resolved from -1 def test_create_default_online_config(self): @@ -183,7 +183,7 @@ def test_create_default_online_config(self): assert isinstance(config, BenchmarkConfig) assert config.settings.load_pattern.type == LoadPatternType.POISSON assert config.settings.load_pattern.target_qps == 10.0 - assert config.settings.timeouts.min_duration_ms == 600000 + assert config.settings.timeouts.min_duration_ms is None def test_create_default_eval_not_implemented(self): with pytest.raises(CLIError, match="EVAL"):