From 17dabb5c97da8eba6afb113e2d4a02b06e552c00 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 10 Jul 2026 17:34:04 -0700 Subject: [PATCH 01/39] refactor(records): remove legacy results processors; route by record type Remove the legacy `results_processor` plugin category entirely and route records dynamically by `record_type` through accumulators and stream exporters. Taxonomy - Delete the `results_processor` category and its registrations. Behavior moves to `accumulator` / `stream_exporter` / `analyzer` / `data_exporter`. - Delete legacy `metric_results_processor`, `timeslice_metric_results_processor`, `record_export_results_processor`, and the bespoke `network_latency/protocols`. - Rename `otel_metrics_results_processor` -> `otel_metrics_streamer` and `accuracy_results_processor` -> the accuracy accumulator. RecordsManager - Replace the per-type processor lists and fan-out methods with a routing table built from plugin metadata and a single `_dispatch_record(record)` keyed on `record.record_type`. DERIVED metrics now resolve in MetricsAccumulator. Accuracy - Move per-record grade transport off the `accuracy.` summary namespace to underscore keys (`accuracy_correct` / `accuracy_unparsed`) so the registered transport metrics no longer collide with the `accuracy.` summary tags (which had been dropping the overall unparsed count). Network-adjusted metrics - Wire `network_adjusted_*` latency metrics into per-timeslice results (they were only in the overall summary). The per-record adjustment `max(latency - rtt, 0)` is window-independent, so it is computed once over each full source column and both the overall summary and every timeslice aggregate masked views -- no redundant per-window subtraction, and no double-application (the analyzer only reads the column store, never writes back). Full unit suite: 14422 passed, 94 skipped, 1 xfailed. Plugin artifacts + schemas validate; every registered plugin class imports. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/accuracy/accuracy_stubs.md | 16 +- docs/cli-options.md | 8 +- docs/dev/patterns.md | 12 +- docs/diagrams/metrics-flow.md | 4 +- docs/plugins/plugin-system.md | 5 +- docs/reference/list-metric-aggregation.md | 2 +- .../accuracy/accuracy_record_processor.py | 14 +- .../accuracy/accuracy_results_processor.py | 24 +- src/aiperf/accuracy/models.py | 12 + .../common/messages/inference_messages.py | 4 +- src/aiperf/common/models/credit_models.py | 3 + .../common/models/network_latency_models.py | 4 + src/aiperf/common/models/record_models.py | 7 +- .../common/models/server_metrics_models.py | 2 + src/aiperf/common/models/telemetry_models.py | 4 + src/aiperf/config/artifacts.py | 2 +- src/aiperf/exporters/metrics_base_exporter.py | 5 +- src/aiperf/gpu_telemetry/protocols.py | 111 +-- src/aiperf/metrics/accumulator.py | 85 +- src/aiperf/metrics/base_aggregate_metric.py | 11 +- src/aiperf/metrics/derived_sum_metric.py | 10 +- src/aiperf/metrics/display_units.py | 35 +- .../metrics/network_adjusted_analyzer.py | 72 +- src/aiperf/metrics/types/accuracy_metrics.py | 45 +- .../metrics/types/network_adjusted_metrics.py | 8 +- .../metrics/types/power_efficiency_metrics.py | 16 +- src/aiperf/network_latency/accumulator.py | 8 +- src/aiperf/network_latency/jsonl_writer.py | 10 +- src/aiperf/network_latency/protocols.py | 30 - .../search_planner/_pooled_percentile.py | 2 +- src/aiperf/plugin/categories.yaml | 27 - src/aiperf/plugin/enums.py | 18 +- src/aiperf/plugin/plugins.py | 15 +- src/aiperf/plugin/plugins.yaml | 116 +-- src/aiperf/plugin/schema/plugins.schema.json | 147 --- .../metric_results_processor.py | 275 ----- .../otel_metrics_results_processor.py | 13 +- src/aiperf/post_processors/protocols.py | 46 +- .../record_export_jsonl_writer.py | 8 - .../record_export_results_processor.py | 93 -- src/aiperf/post_processors/strategies/core.py | 5 +- .../timeslice_metric_results_processor.py | 124 --- src/aiperf/records/records_manager.py | 584 +++-------- .../records/records_manager_processing.py | 32 - src/aiperf/server_metrics/protocols.py | 44 +- .../test_accuracy_record_processor.py | 74 +- .../unit/common/models/test_record_models.py | 103 +- tests/unit/exporters/conftest.py | 94 -- .../test_telemetry_integration.py | 4 +- .../metrics/test_list_metric_aggregation.py | 4 +- .../metrics/test_power_efficiency_metrics.py | 4 +- .../unit/network_latency/test_jsonl_writer.py | 6 +- tests/unit/post_processors/conftest.py | 24 +- .../test_metric_results_processor.py | 490 +++------ .../test_metrics_accumulator.py | 4 +- .../test_network_adjusted_metrics.py | 261 +++-- .../post_processors/test_otel_error_paths.py | 6 +- .../test_otel_metrics_results_processor.py | 28 +- .../test_post_processor_integration.py | 151 ++- .../test_record_export_jsonl_writer.py | 19 +- ...st_record_export_jsonl_writer_detailed.py} | 140 ++- ...test_timeslice_metric_results_processor.py | 469 +++------ .../property/_numeric_bounds_baseline.txt | 1 - tests/unit/records/test_records_manager.py | 939 ++++-------------- .../test_records_manager_network_latency.py | 163 +-- .../test_records_manager_process_results.py | 13 +- .../records/test_records_manager_routing.py | 372 +++---- tests/unit/server_metrics/test_accumulator.py | 4 +- 68 files changed, 1634 insertions(+), 3857 deletions(-) delete mode 100644 src/aiperf/network_latency/protocols.py delete mode 100644 src/aiperf/post_processors/metric_results_processor.py delete mode 100644 src/aiperf/post_processors/record_export_results_processor.py delete mode 100644 src/aiperf/post_processors/timeslice_metric_results_processor.py rename tests/unit/post_processors/{test_record_export_results_processor.py => test_record_export_jsonl_writer_detailed.py} (87%) diff --git a/docs/accuracy/accuracy_stubs.md b/docs/accuracy/accuracy_stubs.md index 32acb9198e..b654e99433 100644 --- a/docs/accuracy/accuracy_stubs.md +++ b/docs/accuracy/accuracy_stubs.md @@ -214,18 +214,18 @@ async def process_record( **File:** `src/aiperf/accuracy/accuracy_results_processor.py` **Parent:** `AIPerfLifecycleMixin` -**Implements:** `ResultsProcessorProtocol` -**Plugin key:** `accuracy_results` (under `results_processor`) +**Implements:** `AccumulatorProtocol` +**Plugin key:** `accuracy_results` (under `accumulator`) **Disables via:** `PostProcessorDisabled` when `not cfg.accuracy.enabled` -This class is fully implemented and serves as the canonical reference for aggregating per-task accuracy metrics. +This class is fully implemented and serves as the canonical reference for aggregating per-task accuracy metrics from routed `metric_records`. ```python -async def process_result(self, record_data: MetricRecordsData) -> None # IMPLEMENTED in PR #815 +async def process_record(self, record_data: MetricRecordsData) -> None # IMPLEMENTED in PR #815 async def summarize(self) -> list[MetricResult] # IMPLEMENTED in PR #815 ``` -**Reference implementation:** `MetricResultsProcessor` in `src/aiperf/post_processors/metric_results_processor.py` +**Reference implementation:** `MetricsAccumulator` in `src/aiperf/metrics/accumulator.py` --- @@ -287,7 +287,7 @@ All stubs are registered in `src/aiperf/plugin/plugins.yaml` and `src/aiperf/plu | Category | Plugin Key | Class | |----------|-----------|-------| | `record_processor` | `accuracy_record` | `AccuracyRecordProcessor` | -| `results_processor` | `accuracy_results` | `AccuracyResultsProcessor` | +| `accumulator` | `accuracy_results` | `AccuracyResultsProcessor` | | `console_exporter` | `accuracy` | `AccuracyConsoleExporter` | | `data_exporter` | `accuracy_csv` | `AccuracyDataExporter` | @@ -302,7 +302,7 @@ All stubs are registered in `src/aiperf/plugin/plugins.yaml` and `src/aiperf/plu | Graders | 7 (all) | 0 | — | 0 | | Benchmarks | 9 (all) | 0 | — | 0 | | Record Processor | 1 (`AccuracyRecordProcessor`) | 0 | — | 0 | -| Results Processor | 1 (`AccuracyResultsProcessor`) | 0 | — | 0 | +| Accuracy Accumulator | 1 (`AccuracyResultsProcessor`) | 0 | — | 0 | | Console Exporter | 1 (`AccuracyConsoleExporter`) | 0 | — | 0 | | Data Exporter | 1 (`AccuracyDataExporter`) | 0 | — | 0 | | Stub-plugin Validator | 1 (`AccuracyConfig._reject_stub_plugins`, idle until next stub) | 0 | — | 0 | @@ -323,7 +323,7 @@ The processors, exporters, all seven graders, and all nine benchmarks are wired | **Canonical grader** | `src/aiperf/accuracy/graders/multiple_choice.py` | | **Canonical benchmark** | `src/aiperf/accuracy/benchmarks/mmlu.py` | | **Canonical record processor** | `src/aiperf/accuracy/accuracy_record_processor.py` | -| **Canonical results processor** | `src/aiperf/accuracy/accuracy_results_processor.py` | +| **Canonical accuracy accumulator** | `src/aiperf/accuracy/accuracy_results_processor.py` | | **Canonical console exporter** | `src/aiperf/accuracy/accuracy_console_exporter.py` | | **Canonical data exporter** | `src/aiperf/accuracy/accuracy_data_exporter.py` | | Disabled exception pattern | `src/aiperf/post_processors/raw_record_writer_processor.py:47` | diff --git a/docs/cli-options.md b/docs/cli-options.md index c070dc311f..29b44a434a 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -265,6 +265,7 @@ aiperf profile --model your_model --url localhost:8000 --goodput "request_latenc #### `-m`, `--model-names`, `--model` `` Model name(s) to be benchmarked. Can be a comma-separated list or a single model name. +
_Default: `[]`_ #### `--model-selection-strategy` `` @@ -428,6 +429,7 @@ HuggingFace dataset subset/config name to override the plugin default (e.g. `sha #### `--dataset-filter` `` Dataset-specific filter in key=value form. Repeat for multiple filters. Only supported by public datasets that declare filter support. +
_Default: `[]`_ #### `--custom-dataset-type` `` @@ -1340,6 +1342,7 @@ Deprecated and ignored. The bayesian preset and the optuna expert mode both use #### `--variant`, `--sweep-variant` `` Repeatable: each occurrence describes one sweep variation. Format: '[name:] key=value, key=value, ...'. Keys are CLI flag names with the leading '--' stripped, in either kebab-case or snake_case (isl, osl, concurrency, request-rate / request_rate, request-count / request_count, benchmark-duration / benchmark_duration, ...). Multi-occurrence emits a ScenarioSweep. Mutually exclusive with magic-list flags, --search-recipe, and YAML-declared sweeps. Single-occurrence is rejected -- use the standalone --isl / --osl / --concurrency flags for a one-off. +
_Default: `[]`_ #### `--search-sla` `` @@ -1642,7 +1645,7 @@ Explore AIPerf plugins: aiperf plugins [category] [type] #### `--category` `` Category to explore. -
_Choices: [`accumulator`, `accuracy_benchmark`, `accuracy_grader`, `api_router`, `arrival_pattern`, `communication`, `communication_client`, `console_exporter`, `convergence_criterion`, `custom_dataset_loader`, `data_exporter`, `dataset_backing_store`, `dataset_client_store`, `dataset_composer`, `dataset_sampler`, `endpoint`, `gpu_telemetry_collector`, `gpu_telemetry_processor`, `plot`, `public_dataset_loader`, `ramp`, `record_processor`, `results_processor`, `search_planner`, `search_recipe`, `search_recipe_post_process`, `server_metrics_processor`, `service`, `service_manager`, `stream_exporter`, `timing_strategy`, `transport`, `ui`, `url_selection_strategy`, `zmq_proxy`]_ +
_Choices: [`accumulator`, `accuracy_benchmark`, `accuracy_grader`, `api_router`, `arrival_pattern`, `communication`, `communication_client`, `console_exporter`, `convergence_criterion`, `custom_dataset_loader`, `data_exporter`, `dataset_backing_store`, `dataset_client_store`, `dataset_composer`, `dataset_sampler`, `endpoint`, `gpu_telemetry_collector`, `plot`, `public_dataset_loader`, `ramp`, `record_processor`, `search_planner`, `search_recipe`, `search_recipe_post_process`, `service`, `service_manager`, `stream_exporter`, `timing_strategy`, `transport`, `ui`, `url_selection_strategy`, `zmq_proxy`]_ #### `--name` `` @@ -1690,6 +1693,7 @@ HTTP port for health endpoints (/healthz, /readyz). Required for Kubernetes live #### `-m`, `--model-names`, `--model` `` Model name(s) to be benchmarked. Can be a comma-separated list or a single model name. +
_Default: `[]`_ #### `--model-selection-strategy` `` @@ -1853,6 +1857,7 @@ HuggingFace dataset subset/config name to override the plugin default (e.g. `sha #### `--dataset-filter` `` Dataset-specific filter in key=value form. Repeat for multiple filters. Only supported by public datasets that declare filter support. +
_Default: `[]`_ #### `--custom-dataset-type` `` @@ -2765,6 +2770,7 @@ Deprecated and ignored. The bayesian preset and the optuna expert mode both use #### `--variant`, `--sweep-variant` `` Repeatable: each occurrence describes one sweep variation. Format: '[name:] key=value, key=value, ...'. Keys are CLI flag names with the leading '--' stripped, in either kebab-case or snake_case (isl, osl, concurrency, request-rate / request_rate, request-count / request_count, benchmark-duration / benchmark_duration, ...). Multi-occurrence emits a ScenarioSweep. Mutually exclusive with magic-list flags, --search-recipe, and YAML-declared sweeps. Single-occurrence is rejected -- use the standalone --isl / --osl / --concurrency flags for a one-off. +
_Default: `[]`_ #### `--search-sla` `` diff --git a/docs/dev/patterns.md b/docs/dev/patterns.md index a89d9a6d2e..790267a896 100644 --- a/docs/dev/patterns.md +++ b/docs/dev/patterns.md @@ -450,7 +450,7 @@ class TotalGpuEnergyMetric(BaseDerivedMetric[float]): Invariant: externally injected by `GPUTelemetryAccumulator.compute_efficiency_metrics` from energy_consumption counter deltas. `_derive_value` is intentionally - non-functional; `MetricResultsProcessor.update_derived_metrics` is + non-functional; `MetricsAccumulator._resolve_derived_metrics` is expected to catch NoMetricValue and skip the tag during its derivation walk. """ @@ -472,7 +472,7 @@ def _derive_value(self, metric_results: MetricResultsDict) -> NoReturn: "is externally injected by " "GPUTelemetryAccumulator.compute_efficiency_metrics. If this exception " "surfaces, the derivation walk is missing its NoMetricValue handler " - "(see MetricResultsProcessor.update_derived_metrics)." + "(see MetricsAccumulator._resolve_derived_metrics)." ) ``` @@ -486,7 +486,7 @@ is enforced. The recommended shape is: - *Injection site*: which method is the source of truth (`GPUTelemetryAccumulator.compute_efficiency_metrics`). - *Catching path*: where the exception is expected to be absorbed - (`MetricResultsProcessor.update_derived_metrics`). If this fires in + (`MetricsAccumulator._resolve_derived_metrics`). If this fires in production, the catching path has a bug. ### Why not just skip the class entirely? @@ -503,9 +503,9 @@ external injection. `GPUTelemetryAccumulator.compute_efficiency_metrics`, which constructs `MetricResult` objects directly with the relevant tags and appends them to the records list before `ProcessRecordsResult` is built. The standard -`update_derived_metrics` walk sees these tags too, raises `NoMetricValue` -via `_derive_value`, catches it, and skips — so the externally-injected -values are not overwritten. +`MetricsAccumulator._resolve_derived_metrics` walk sees these tags too, +raises `NoMetricValue` via `_derive_value`, catches it, and skips — so the +externally-injected values are not overwritten. ### Test contract diff --git a/docs/diagrams/metrics-flow.md b/docs/diagrams/metrics-flow.md index 13694b1020..30c7cf82b4 100644 --- a/docs/diagrams/metrics-flow.md +++ b/docs/diagrams/metrics-flow.md @@ -32,8 +32,8 @@ flowchart TD C3 --> E3["MetricRecordDict
(Per-record results)"] D3 --> E3 - %% Stage 2: Centralized Results Processing - E1 --> G["RecordsManager → MetricResultsProcessor
(Single centralized instance)"] + %% Stage 2: Centralized Metrics Accumulation + E1 --> G["RecordsManager → MetricsAccumulator
(Single centralized instance)"] E2 --> G E3 --> G diff --git a/docs/plugins/plugin-system.md b/docs/plugins/plugin-system.md index b26be93ed6..58c0c22864 100644 --- a/docs/plugins/plugin-system.md +++ b/docs/plugins/plugin-system.md @@ -134,9 +134,8 @@ AIPerf supports 33 plugin categories organized by function, including `api_route | Category | Enum | Description | |----------|------|-------------| | `record_processor` | `RecordProcessorType` | Per-record metric computation | -| `results_processor` | `ResultsProcessorType` | Aggregated results computation | -| `gpu_telemetry_processor` | `GPUTelemetryProcessorType` | Side-channel GPU telemetry aggregation/export within `GPUTelemetryManager` | -| `server_metrics_processor` | `ServerMetricsProcessorType` | Side-channel Prometheus server metrics aggregation/export within `ServerMetricsManager` | +| `accumulator` | `AccumulatorType` | Record-type-routed aggregation and summary computation | +| `stream_exporter` | `StreamExporterType` | Record-type-routed streaming sinks such as JSONL and OpenTelemetry | | `data_exporter` | `DataExporterType` | File format exporters (CSV, JSON, Parquet) | | `console_exporter` | `ConsoleExporterType` | Terminal output exporters | diff --git a/docs/reference/list-metric-aggregation.md b/docs/reference/list-metric-aggregation.md index e40d78fde0..2f75fb9851 100644 --- a/docs/reference/list-metric-aggregation.md +++ b/docs/reference/list-metric-aggregation.md @@ -56,6 +56,6 @@ For all other metrics: **no change**. Scalar record metrics still use the exact- ## Where it lives - Aggregator class: [`src/aiperf/metrics/list_metric_aggregation.py`](https://github.com/ai-dynamo/aiperf/blob/main/src/aiperf/metrics/list_metric_aggregation.py) — `TDigestListMetricAggregator`. -- Selection site: [`src/aiperf/post_processors/metric_results_processor.py`](https://github.com/ai-dynamo/aiperf/blob/main/src/aiperf/post_processors/metric_results_processor.py) — first-touch dispatch by `isinstance(value, list)`. +- Selection site: [`src/aiperf/metrics/column_store.py`](https://github.com/ai-dynamo/aiperf/blob/main/src/aiperf/metrics/column_store.py) — first-touch dispatch by `isinstance(value, list)` in `ColumnStore._resolve_tag_handler()`. - Compression knob: `Environment.METRICS.TDIGEST_COMPRESSION` (env: `AIPERF_METRICS_TDIGEST_COMPRESSION`, default 500). - Dependency: [`crick~=0.0.8`](https://pypi.org/project/crick/) (Cython/C-backed t-digest, BSD-3, dask-org maintained). diff --git a/src/aiperf/accuracy/accuracy_record_processor.py b/src/aiperf/accuracy/accuracy_record_processor.py index f90e6c7d2e..38c9f73f1e 100644 --- a/src/aiperf/accuracy/accuracy_record_processor.py +++ b/src/aiperf/accuracy/accuracy_record_processor.py @@ -5,7 +5,11 @@ from typing import TYPE_CHECKING, Any -from aiperf.accuracy.models import GradingResult +from aiperf.accuracy.models import ( + ACCURACY_RECORD_CORRECT_KEY, + ACCURACY_RECORD_UNPARSED_KEY, + GradingResult, +) from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.mixins import AIPerfLifecycleMixin from aiperf.common.models import MetricRecordMetadata, ParsedResponseRecord @@ -77,7 +81,9 @@ async def process_record( Maps ``metadata.session_num % len(_ground_truths)`` to the ground-truth answer, runs the configured grader, and returns a MetricRecordDict - containing ``accuracy.correct`` and ``accuracy.unparsed``. + containing the per-record transport keys ``accuracy_correct`` and + ``accuracy_unparsed`` (underscore-namespaced so they never collide with + the ``accuracy.`` summary tags). Raises: RuntimeError: if on_dataset_configured was not called before processing. @@ -96,8 +102,8 @@ async def process_record( result: GradingResult = await self.grader.grade(response_text, ground_truth) - record_metrics["accuracy.correct"] = 1.0 if result.correct else 0.0 - record_metrics["accuracy.unparsed"] = 1.0 if result.unparsed else 0.0 + record_metrics[ACCURACY_RECORD_CORRECT_KEY] = 1.0 if result.correct else 0.0 + record_metrics[ACCURACY_RECORD_UNPARSED_KEY] = 1.0 if result.unparsed else 0.0 self._log_grading_detail(metadata.session_num, response_text, result) diff --git a/src/aiperf/accuracy/accuracy_results_processor.py b/src/aiperf/accuracy/accuracy_results_processor.py index 244e03e2d6..88d216680e 100644 --- a/src/aiperf/accuracy/accuracy_results_processor.py +++ b/src/aiperf/accuracy/accuracy_results_processor.py @@ -8,10 +8,13 @@ from aiperf.accuracy.models import ( ACCURACY_OVERALL_TAG, + ACCURACY_RECORD_CORRECT_KEY, + ACCURACY_RECORD_UNPARSED_KEY, ACCURACY_UNPARSED_TAG, accuracy_task_tag, accuracy_unparsed_task_tag, ) +from aiperf.common.enums import MetricConsoleGroup from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.mixins import AIPerfLifecycleMixin from aiperf.common.models import MetricResult @@ -54,7 +57,7 @@ def on_dataset_configured(self, metadata: DatasetMetadata) -> None: Called by RecordsManager before any records are processed. Builds the ordered list of task names from ConversationMetadata so that - process_result can bucket results without re-loading the benchmark. + process_record can bucket records without re-loading the benchmark. """ self._tasks = [ c.accuracy_task @@ -62,12 +65,12 @@ def on_dataset_configured(self, metadata: DatasetMetadata) -> None: if c.accuracy_task is not None ] - async def process_result(self, record_data: MetricRecordsData) -> None: + async def process_record(self, record_data: MetricRecordsData) -> None: """Accumulate per-task accuracy counts from a single record's metrics. - Reads ``accuracy.correct`` from ``record_data.metrics`` (produced by + Reads ``accuracy_correct`` from ``record_data.metrics`` (produced by AccuracyRecordProcessor) and increments per-task and overall counters. - Records missing the ``accuracy.correct`` key are silently skipped. + Records missing the ``accuracy_correct`` key are silently skipped. Raises: RuntimeError: if on_dataset_configured was not called before processing. @@ -75,16 +78,16 @@ async def process_result(self, record_data: MetricRecordsData) -> None: if self._tasks is None: raise RuntimeError( "AccuracyResultsProcessor: dataset not configured; " - "on_dataset_configured must be called before process_result" + "on_dataset_configured must be called before process_record" ) metrics = record_data.metrics - correct = metrics.get("accuracy.correct") + correct = metrics.get(ACCURACY_RECORD_CORRECT_KEY) if correct is None: return task = self._tasks[record_data.metadata.session_num % len(self._tasks)] is_correct = float(correct) >= 0.5 - is_unparsed = float(metrics.get("accuracy.unparsed", 0.0)) >= 0.5 + is_unparsed = float(metrics.get(ACCURACY_RECORD_UNPARSED_KEY, 0.0)) >= 0.5 self._overall_total += 1 if is_correct: @@ -121,6 +124,10 @@ async def summarize(self) -> list[MetricResult]: count=self._overall_total, current=overall_acc, sum=self._overall_correct, + # Rendered by the dedicated Accuracy Benchmark Results table, + # not the main LLM metrics table (a ratio has no avg/p99/etc, + # so it would show as a row of N/A there). + console_group=MetricConsoleGroup.NONE, ) ) @@ -136,6 +143,7 @@ async def summarize(self) -> list[MetricResult]: count=total, current=acc, sum=correct, + console_group=MetricConsoleGroup.NONE, ) ) @@ -148,6 +156,7 @@ async def summarize(self) -> list[MetricResult]: count=self._overall_total, current=self._overall_unparsed / self._overall_total, sum=self._overall_unparsed, + console_group=MetricConsoleGroup.NONE, ) ) @@ -162,6 +171,7 @@ async def summarize(self) -> list[MetricResult]: count=total, current=unparsed / total if total > 0 else 0.0, sum=unparsed, + console_group=MetricConsoleGroup.NONE, ) ) diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index 6933bf22e8..f0ce40f914 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -8,12 +8,24 @@ from aiperf.common.models.base_models import AIPerfBaseModel +# Summary/metric tags (the ``accuracy.`` dot namespace) emitted by +# AccuracyResultsProcessor.summarize() and read by the accuracy exporters. ACCURACY_OVERALL_TAG = "accuracy.overall" ACCURACY_TASK_TAG_PREFIX = "accuracy.task." ACCURACY_UNPARSED_TAG = "accuracy.unparsed" ACCURACY_UNPARSED_TASK_TAG_PREFIX = "accuracy.unparsed.task." ACCURACY_METRIC_PREFIX = "accuracy." +# Per-record TRANSPORT keys (NOT metric tags): AccuracyRecordProcessor writes +# these into each record's MetricRecordDict to hand the grade to +# AccuracyResultsProcessor.process_record. They use an ``accuracy_`` (underscore) +# prefix so they live outside the ``accuracy.`` (dot) summary namespace and can +# never be mistaken for -- or collide with -- a summary/metric tag. Keeping them +# in the dot namespace is what let the registered ``accuracy.unparsed`` transport +# metric shadow the ``accuracy.unparsed`` summary tag; the underscore separates them. +ACCURACY_RECORD_CORRECT_KEY = "accuracy_correct" +ACCURACY_RECORD_UNPARSED_KEY = "accuracy_unparsed" + def accuracy_task_tag(task: str) -> str: """Build the MetricResult.tag for a per-task accuracy result.""" diff --git a/src/aiperf/common/messages/inference_messages.py b/src/aiperf/common/messages/inference_messages.py index 141a280c5c..ea0eb4d60a 100644 --- a/src/aiperf/common/messages/inference_messages.py +++ b/src/aiperf/common/messages/inference_messages.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Any +from typing import Any, ClassVar from pydantic import Field, SerializeAsAny, field_validator @@ -30,6 +30,8 @@ class InferenceResultsMessage(BaseServiceMessage): class MetricRecordsData(AIPerfBaseModel): """Incoming data from the record processor service to combine metric records for the profile run.""" + record_type: ClassVar[str] = "metric_records" + metadata: MetricRecordMetadata = Field( ..., description="The metadata of the request record." ) diff --git a/src/aiperf/common/models/credit_models.py b/src/aiperf/common/models/credit_models.py index 737c31e08f..6e33b41c9a 100644 --- a/src/aiperf/common/models/credit_models.py +++ b/src/aiperf/common/models/credit_models.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import time +from typing import ClassVar from pydantic import ConfigDict, Field @@ -123,6 +124,8 @@ def is_requests_complete(self) -> bool: class CreditPhaseStats(BasePhaseStats): """Immutable model for phase credit stats. This is used to track the progress of the credit phases.""" + record_type: ClassVar[str] = "credit_phase_stats" + model_config = ConfigDict(frozen=True) # Credit progress fields diff --git a/src/aiperf/common/models/network_latency_models.py b/src/aiperf/common/models/network_latency_models.py index 72ce466847..1f9bf2f74c 100644 --- a/src/aiperf/common/models/network_latency_models.py +++ b/src/aiperf/common/models/network_latency_models.py @@ -3,6 +3,8 @@ from __future__ import annotations +from typing import ClassVar + from pydantic import Field from aiperf.common.models.base_models import AIPerfBaseModel @@ -23,6 +25,8 @@ class NetworkLatencySample(AIPerfBaseModel): ``success=False``, ``rtt_ns=None`` and the captured ``error``. """ + record_type: ClassVar[str] = "network_latency" + timestamp_ns: int = Field( ge=0, description="Wall-clock timestamp (time.time_ns) when the probe was issued", diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index 29877a5395..fb34dceaa7 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -39,7 +39,7 @@ from aiperf.common.models.model_endpoint_info import ModelEndpointInfo from aiperf.common.models.trace_models import BaseTraceData, TraceDataExport from aiperf.common.models.usage_models import Usage -from aiperf.common.types import JsonObject, MetricTagT, TimeSliceT +from aiperf.common.types import JsonObject, MetricTagT from aiperf.common.utils import load_json_str _logger = AIPerfLogger(__name__) @@ -270,11 +270,6 @@ class ProfileResults(AIPerfBaseModel): "with its metric results. Position in the list is the slice's " "chronological index. Produced by the MetricsAccumulator engine.", ) - timeslice_metric_results: dict[TimeSliceT, list[MetricResult]] | None = Field( - default=None, - description="The timeslice metric results of the profile (if using the " - "legacy timeslice results-processor path)", - ) total_expected: int | None = Field( default=None, description="The total number of inference requests expected to be made (if known)", diff --git a/src/aiperf/common/models/server_metrics_models.py b/src/aiperf/common/models/server_metrics_models.py index e6766e7b07..692447177b 100644 --- a/src/aiperf/common/models/server_metrics_models.py +++ b/src/aiperf/common/models/server_metrics_models.py @@ -215,6 +215,8 @@ class ServerMetricsRecord(AIPerfBaseModel): the client received the full response. """ + record_type: ClassVar[str] = "server_metrics" + endpoint_url: str = Field( description="Source Prometheus metrics endpoint URL (e.g., 'http://localhost:8081/metrics')" ) diff --git a/src/aiperf/common/models/telemetry_models.py b/src/aiperf/common/models/telemetry_models.py index 425e202761..375cb3d1c3 100644 --- a/src/aiperf/common/models/telemetry_models.py +++ b/src/aiperf/common/models/telemetry_models.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from typing import ClassVar + import numpy as np from numpy.typing import NDArray from pydantic import ConfigDict, Field @@ -153,6 +155,8 @@ class TelemetryRecord(GpuMetadata): Inherits from GpuMetadata to avoid duplicating metadata fields. """ + record_type: ClassVar[str] = "gpu_telemetry" + timestamp_ns: int = Field( description="Nanosecond wall-clock timestamp when telemetry was collected (time_ns)" ) diff --git a/src/aiperf/config/artifacts.py b/src/aiperf/config/artifacts.py index 8f797a6f15..27462e4994 100644 --- a/src/aiperf/config/artifacts.py +++ b/src/aiperf/config/artifacts.py @@ -32,7 +32,7 @@ # Type aliases for format arrays. # Narrow to what the codebase actually emits: MetricsJsonExporter writes the -# summary JSON, RecordExportResultsProcessor writes the records JSONL. No YAML +# summary JSON, RecordExportJSONLWriter writes the records JSONL. No YAML # summary exporter and no records-CSV exporter exist; do not advertise them. SummaryExportFormat = Literal["json"] RecordsExportFormat = Literal["jsonl"] diff --git a/src/aiperf/exporters/metrics_base_exporter.py b/src/aiperf/exporters/metrics_base_exporter.py index 1345c62d4a..9240024a86 100644 --- a/src/aiperf/exporters/metrics_base_exporter.py +++ b/src/aiperf/exporters/metrics_base_exporter.py @@ -35,9 +35,8 @@ def _prepare_metrics( INTERNAL and EXPERIMENTAL metrics are computed (they may be dependencies of other metrics) but excluded from file exports unless the dev-mode show - flags are set -- mirroring the console exporter's ``exclude_flags`` and - the legacy ``MetricResultsProcessor.summarize`` filter. The accumulator - summary engine does NOT pre-filter, so the file exporters must. Tags not + flags are set -- mirroring the console exporter's ``exclude_flags``. + The accumulator summary engine does NOT pre-filter, so the file exporters must. Tags not in ``MetricRegistry`` (dynamically injected, e.g. derived latency) are kept. diff --git a/src/aiperf/gpu_telemetry/protocols.py b/src/aiperf/gpu_telemetry/protocols.py index 5cf85c8067..b10c07ebe8 100644 --- a/src/aiperf/gpu_telemetry/protocols.py +++ b/src/aiperf/gpu_telemetry/protocols.py @@ -14,16 +14,11 @@ MetricResult, TelemetryExportData, ) - from aiperf.common.models.server_metrics_models import TimeRangeFilter @runtime_checkable class GPUTelemetryCollectorProtocol(Protocol): - """Protocol for GPU telemetry collectors. - - Defines the interface for collectors that gather GPU metrics from various sources - (DCGM HTTP endpoints, pynvml library, etc.) and deliver them via callbacks. - """ + """Protocol for GPU telemetry collectors.""" @property def id(self) -> str: @@ -48,122 +43,42 @@ async def stop(self) -> None: ... async def is_url_reachable(self) -> bool: - """Check if the collector source is available. - - For DCGM: Tests HTTP endpoint reachability. - For pynvml: Tests NVML library initialization. - - Returns: - True if the source is available and ready for collection. - """ + """Check if the collector source is available.""" ... async def collect_and_process_metrics(self) -> None: - """Perform a one-shot scrape and dispatch records via the configured callback. - - Called by ``GPUTelemetryManager`` for baseline and final-state capture, - outside the collector's own periodic background task. Implementations - must be safe to invoke before ``start()`` (i.e. after ``initialize()``) - and concurrently with the periodic loop. - """ + """Perform a one-shot scrape and dispatch records via the configured callback.""" ... @classmethod def validate_environment(cls) -> None: - """Raise RuntimeError if this collector cannot run on the current host. - - Called during :class:`GpuTelemetryConfig` validation for local - collectors before the benchmark starts so missing native bindings - or required system libraries produce a friendly CLI error rather - than a runtime traceback. Remote collectors (e.g. DCGM) implement - this as a no-op. - """ + """Raise RuntimeError if this collector cannot run on the current host.""" ... -# Type aliases for callbacks TRecordCallback = Callable[[list[TelemetryRecord], str], Awaitable[None]] TErrorCallback = Callable[[ErrorDetails, str], Awaitable[None]] @runtime_checkable -class GPUTelemetryProcessorProtocol(Protocol): - """Protocol for GPU telemetry results processors that handle TelemetryRecord objects. - - This protocol is separate from ResultsProcessorProtocol because GPU telemetry data - has fundamentally different structure (hierarchical with metadata) compared - to inference metrics (flat key-value pairs). - """ +class GPUTelemetryAccumulatorProtocol(Protocol): + """Protocol for GPU telemetry accumulators and realtime telemetry.""" - async def process_telemetry_record(self, record: TelemetryRecord) -> None: - """Process individual telemetry record with rich metadata. - - Args: - record: TelemetryRecord containing GPU metrics and hierarchical metadata - """ + async def process_record(self, record: TelemetryRecord) -> None: + """Process one GPU telemetry sample.""" ... - -@runtime_checkable -class GPUTelemetryAccumulatorProtocol(GPUTelemetryProcessorProtocol, Protocol): - """Protocol for GPU telemetry accumulators that accumulate GPU telemetry data and export pre-computed metrics. - - Extends GPUTelemetryProcessorProtocol to provide result export, realtime telemetry, and summarization - capabilities. Implementations should accumulate DCGM metrics, compute aggregated statistics per GPU, - and support dynamic dashboard enablement for realtime monitoring. - """ + async def summarize(self) -> list[MetricResult]: ... def export_results( self, - start_ns: int, - end_ns: int, + start_ns: int | None = None, + end_ns: int | None = None, error_summary: list[ErrorDetailsCount] | None = None, ) -> TelemetryExportData | None: - """Export accumulated telemetry data as a TelemetryExportData object. - - Args: - start_ns: Start time of collection in nanoseconds - end_ns: End time of collection in nanoseconds - error_summary: Optional list of error counts - - Returns: - TelemetryExportData object with pre-computed metrics for each GPU - """ + """Export accumulated telemetry data.""" ... def start_realtime_telemetry(self) -> None: - """Start the realtime telemetry background task. - - This is called when the user dynamically enables the telemetry dashboard - by pressing the telemetry option in the UI without having passed the 'dashboard' parameter - at startup. - """ - - def compute_efficiency_metrics( - self, - metric_results: list[MetricResult], - time_filter: TimeRangeFilter, - ) -> list[MetricResult]: - """Compute cross-boundary power efficiency metrics. - - Args: - metric_results: All metric results from the profiling phase. - time_filter: Time range covering the profiling phase. - - Returns: - Up to three MetricResult objects covering total GPU power, total - GPU energy, and output tokens per joule. Each metric is - independently omitted when its underlying GPU signal is - unavailable; returns an empty list when no GPU has any of the - relevant signals. - """ - ... - - async def summarize(self) -> list[MetricResult]: - """Generate MetricResult list with hierarchical tags for telemetry data. - - Returns: - List of MetricResult objects with hierarchical tags that preserve - dcgm_url -> gpu_uuid grouping structure for dashboard filtering. - """ + """Start realtime telemetry publishing.""" ... diff --git a/src/aiperf/metrics/accumulator.py b/src/aiperf/metrics/accumulator.py index d74786bec2..f2c00f8449 100644 --- a/src/aiperf/metrics/accumulator.py +++ b/src/aiperf/metrics/accumulator.py @@ -14,9 +14,11 @@ from aiperf.common.constants import NANOS_PER_SECOND from aiperf.common.enums import ( AggregationKind, + MetricFlags, MetricType, MetricValueTypeT, ) +from aiperf.common.environment import Environment from aiperf.common.exceptions import NoMetricValue from aiperf.common.messages import MetricRecordsData from aiperf.common.models import MetricResult, TimesliceResult @@ -32,7 +34,10 @@ from aiperf.metrics.display_units import to_display_unit from aiperf.metrics.metric_dicts import MetricResultsDict, metric_result_from_array from aiperf.metrics.metric_registry import MetricRegistry -from aiperf.metrics.network_adjusted_analyzer import inject_network_adjusted_metrics +from aiperf.metrics.network_adjusted_analyzer import ( + compute_network_adjusted_arrays, + inject_network_adjusted_from_arrays, +) from aiperf.post_processors.base_metrics_processor import BaseMetricsProcessor if TYPE_CHECKING: @@ -51,6 +56,17 @@ } +class _MetricClassLookup: + def __init__(self, metric_classes: dict[MetricTagT, Any]) -> None: + self._metric_classes = metric_classes + + def get_class(self, tag: MetricTagT) -> Any: + metric_class = self._metric_classes.get(tag) + if metric_class is None: + raise KeyError(tag) + return metric_class + + class MetricsAccumulator(BaseMetricsProcessor): """Numpy-backed accumulator for inference metrics. @@ -392,14 +408,42 @@ def compute_results_for_mask( ) return self._convert_display_units(raw) - @staticmethod def _convert_display_units( + self, results: dict[MetricTagT, MetricResult], ) -> dict[MetricTagT, MetricResult]: """Convert all metric results from native units to display units.""" + registry = _MetricClassLookup(self._metric_classes) return { - tag: to_display_unit(result, MetricRegistry) + tag: to_display_unit(result, registry) for tag, result in results.items() + } + + def _should_include_in_summary(self, tag: MetricTagT) -> bool: + """Return False for hidden internal/experimental metrics.""" + metric_class = self._metric_classes.get(tag) + if metric_class is None: + return True + has_flags = getattr(metric_class, "has_flags", None) + if not callable(has_flags): + return True + if ( + has_flags(MetricFlags.INTERNAL) + and not Environment.DEV.SHOW_INTERNAL_METRICS + ): + return False + return not ( + has_flags(MetricFlags.EXPERIMENTAL) + and not Environment.DEV.SHOW_EXPERIMENTAL_METRICS + ) + + def _filter_hidden_metrics( + self, results: dict[MetricTagT, MetricResult] + ) -> dict[MetricTagT, MetricResult]: + """Drop computed metrics that should not appear in summary exports.""" + return { + tag: result for tag, result in results.items() + if self._should_include_in_summary(tag) } def set_network_rtt_ns(self, rtt_ns: float | None) -> None: @@ -445,6 +489,7 @@ def _summarize_for_export_context( ) timeslices: list[TimesliceResult] | None = None + adjusted_arrays: dict[str, FloatArray] | None = None has_records = self._column_store.count > 0 and ( mask is None or bool(mask.any()) @@ -458,8 +503,18 @@ def _summarize_for_export_context( window_start_ns=window_start_ns, window_end_ns=window_end_ns, ) + # Network-RTT-adjusted latency: the per-record subtraction is + # window-independent, so compute the clamped arrays ONCE here and let + # the overall summary and every timeslice aggregate masked views. No-op + # unless the RecordsManager delivered a (truthy) RTT via set_network_rtt_ns. + if self._network_rtt_ns: + adjusted_arrays = compute_network_adjusted_arrays( + self._column_store, self._network_rtt_ns + ) if self._slice_duration_ns is not None: - timeslices = self._compute_timeslices(sweeps, mask=mask) + timeslices = self._compute_timeslices( + sweeps, mask=mask, adjusted_arrays=adjusted_arrays + ) overall_results = self._convert_display_units(overall_results) @@ -469,17 +524,15 @@ def _summarize_for_export_context( inject_derived_latency_metrics( self._column_store, overall_results, mask=mask ) - # Network-RTT-adjusted latency metrics — subtract the run-level mean - # RTT from the request-start-anchored latency arrays. No-op unless the - # RecordsManager delivered a (truthy) RTT via set_network_rtt_ns. - if self._network_rtt_ns: - inject_network_adjusted_metrics( - self._column_store, + if adjusted_arrays is not None: + inject_network_adjusted_from_arrays( + adjusted_arrays, overall_results, self._network_rtt_ns, mask=mask, ) + overall_results = self._filter_hidden_metrics(overall_results) self.debug(lambda: f"Summarized {len(overall_results)} metric results") return AccumulatorMetricsSummary( results=overall_results, @@ -521,6 +574,7 @@ def _compute_timeslices( self, sweeps: Any, mask: BoolArray | None = None, + adjusted_arrays: dict[str, FloatArray] | None = None, ) -> list[TimesliceResult]: """Compute per-timeslice results by partitioning the time range. @@ -602,6 +656,17 @@ def _compute_timeslices( continue results.update(sweeps.compute_metrics(window_start, window_end)) results = self._convert_display_units(results) + # Network-RTT-adjusted latency metrics per window, aggregated from the + # arrays precomputed once in summarize() — this window just slices its + # records out via full_mask. None unless a run-level RTT was delivered. + if adjusted_arrays is not None: + inject_network_adjusted_from_arrays( + adjusted_arrays, + results, + self._network_rtt_ns, + mask=full_mask, + ) + results = self._filter_hidden_metrics(results) timeslices.append( TimesliceResult( start_ns=int(window_start), diff --git a/src/aiperf/metrics/base_aggregate_metric.py b/src/aiperf/metrics/base_aggregate_metric.py index c8314c0664..4f9b8f5214 100644 --- a/src/aiperf/metrics/base_aggregate_metric.py +++ b/src/aiperf/metrics/base_aggregate_metric.py @@ -20,13 +20,10 @@ class BaseAggregateMetric( For each distributed RecordProcessor, an instance of this class is created. This instance is passed the record and the existing record metrics, and is responsible for returning the individual value for that record. It should not use or update the aggregate value here. - The ResultsProcessor creates a singleton instance of this class, which will be used to aggregate the results from the distributed - RecordProcessors. It calls the `_aggregate_value` method, which each metric class must implement to define how values from different - processes are aggregated, such as summing the values, or taking the min/max/average, etc. - - Subclasses declare ``aggregation_kind`` (SUM/MAX/MIN) so that vectorized - accumulators (``MetricsAccumulator``) can fold per-record values into a - single scalar without replaying ``_aggregate_value``. The default is SUM. + The MetricsAccumulator folds the emitted per-record aggregate values into a + single scalar. Subclasses declare ``aggregation_kind`` (SUM/MAX/MIN) so the + vectorized path can combine values without replaying ``_aggregate_value``. + The default is SUM. Examples: ```python diff --git a/src/aiperf/metrics/derived_sum_metric.py b/src/aiperf/metrics/derived_sum_metric.py index 9ca2a04617..52c710772c 100644 --- a/src/aiperf/metrics/derived_sum_metric.py +++ b/src/aiperf/metrics/derived_sum_metric.py @@ -56,12 +56,10 @@ def _derive_value(cls, metric_results: MetricResultsDict) -> MetricValueTypeVarT value = metric_results.get(cls.record_metric_type.tag) if value is None: raise ValueError(f"{cls.record_metric_type.tag} is missing in the metrics.") - # Two engines feed this: the MetricsAccumulator pipeline already stores - # the running-sum scalar in ``scalar_dict[tag]`` (see - # ``MetricsAccumulator._collect_scalars_and_arrays``), so the value - # already IS the sum. The legacy ``MetricResultsProcessor`` (kept alive - # for the otel / timeslice side-channel) instead stores a - # ``MetricAggregator`` (e.g. ``MetricArray``) whose ``.sum`` must be read. + # MetricsAccumulator stores the running-sum scalar in ``scalar_dict[tag]`` + # (see ``MetricsAccumulator._collect_scalars_and_arrays``), so the value + # already IS the sum. The ``MetricAggregator`` branch supports direct unit + # tests and older callers that still pass a live aggregate container. if isinstance(value, MetricAggregator): return value.sum return value diff --git a/src/aiperf/metrics/display_units.py b/src/aiperf/metrics/display_units.py index f0768bc203..07f55a0369 100644 --- a/src/aiperf/metrics/display_units.py +++ b/src/aiperf/metrics/display_units.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from typing import Any + from aiperf.common.aiperf_logger import AIPerfLogger from aiperf.common.constants import STAT_KEYS from aiperf.common.exceptions import MetricTypeError, MetricUnitError @@ -24,19 +26,29 @@ def to_display_unit(result: MetricResult, registry: MetricRegistry) -> MetricRes metric_cls = _resolve_metric_class(registry, result.tag) if metric_cls is None: return result - if result.unit and result.unit != metric_cls.unit.value: + + unit = getattr(metric_cls, "unit", None) + unit_value = _unit_value(unit) + if unit_value is None: + return result + + if result.unit and result.unit != unit_value: _logger.error( - f"Metric {result.tag} has a unit ({result.unit}) that does not match the expected unit ({metric_cls.unit.value}). " - f"({metric_cls.unit.value}) will be used for conversion." + f"Metric {result.tag} has a unit ({result.unit}) that does not match the expected unit ({unit_value}). " + f"({unit_value}) will be used for conversion." ) - display_unit = metric_cls.display_unit or metric_cls.unit + display_unit = getattr(metric_cls, "display_unit", None) or unit + display_unit_value = _unit_value(display_unit) + if display_unit_value is None or display_unit == unit: + return result - if display_unit == metric_cls.unit: + convert_to = getattr(unit, "convert_to", None) + if not callable(convert_to): return result record = result.model_copy(deep=True) - record.unit = display_unit.value + record.unit = display_unit_value for stat in STAT_KEYS: val = getattr(record, stat, None) @@ -48,17 +60,22 @@ def to_display_unit(result: MetricResult, registry: MetricRegistry) -> MetricRes # required — the convert_to call returns ``inf`` unchanged. if isinstance(val, int | float): try: - new_value = metric_cls.unit.convert_to(display_unit, val) + new_value = convert_to(display_unit, val) except MetricUnitError as e: _logger.warning( - f"Error converting {stat} for {result.tag} from {metric_cls.unit.value} to {display_unit.value}: {e}" + f"Error converting {stat} for {result.tag} from {unit_value} to {display_unit_value}: {e}" ) continue setattr(record, stat, new_value) return record -def _resolve_metric_class(registry: MetricRegistry, tag: str): +def _unit_value(unit: Any) -> str | None: + value = getattr(unit, "value", None) + return value if isinstance(value, str) else None + + +def _resolve_metric_class(registry: MetricRegistry, tag: str) -> Any | None: """Look up the metric class for ``tag``, falling back to the parent tag for ``adj_`` synthetic derived metrics so they inherit unit metadata.""" try: diff --git a/src/aiperf/metrics/network_adjusted_analyzer.py b/src/aiperf/metrics/network_adjusted_analyzer.py index 45576e8efe..000c644f25 100644 --- a/src/aiperf/metrics/network_adjusted_analyzer.py +++ b/src/aiperf/metrics/network_adjusted_analyzer.py @@ -10,9 +10,8 @@ Subtracting a constant RTT shifts the mean and every percentile by that constant and leaves the standard deviation unchanged; the per-record subtraction is clamped at 0 so a measured RTT larger than a (rare) sub-RTT latency cannot go -negative. This mirrors the legacy ``MetricResultsProcessor`` injection, but reads -the columnar source arrays the accumulator already owns rather than re-aggregating -a separate ``MetricArray``. +negative. This reads the columnar source arrays the accumulator already owns, +without re-aggregating a separate metric container. Inter-token / inter-chunk latencies are intentionally NOT adjusted: the RTT cancels in ``(request_latency - ttft)``, so those metrics are already @@ -81,24 +80,10 @@ def _array_to_metric_result( ) -def inject_network_adjusted_metrics( - store: ColumnStore, - results: dict[str, MetricResult], - rtt_ns: float, - mask: NDArray[np.bool_] | None = None, -) -> None: - """Inject ``network_adjusted_*`` distributions and the ``network_rtt`` scalar. - - ``rtt_ns`` is the run-level mean network RTT (nanoseconds) resolved by the - RecordsManager (manual ``--network-latency-mean`` override or the mean over - successful probe samples). Caller guarantees ``rtt_ns`` is truthy; a 0/None - RTT is a no-op handled upstream. ``mask`` restricts the distributions to the - export window's records (e.g. a phase-scoped export); None means all records. - """ +def _network_rtt_result(rtt_ns: float) -> MetricResult: + """Build the single run-level ``network_rtt`` scalar MetricResult (ms).""" rtt_ms = rtt_ns / _NS_PER_MS - - # network_rtt: single run-level scalar (no per-record distribution). - results[NetworkRttMetric.tag] = MetricResult( + return MetricResult( tag=NetworkRttMetric.tag, header=NetworkRttMetric.header, unit="ms", @@ -120,16 +105,47 @@ def inject_network_adjusted_metrics( console_group=MetricConsoleGroup.DEFAULT, ) - for adjusted_tag, source_tag in NETWORK_ADJUSTED_SOURCES.items(): - source_ns = store.numeric(source_tag) - if mask is not None: - source_ns = source_ns[mask] - valid_ns = source_ns[~np.isnan(source_ns)] - if valid_ns.size == 0: + +def compute_network_adjusted_arrays( + store: ColumnStore, rtt_ns: float +) -> dict[str, NDArray[np.float64]]: + """Compute the clamped per-record network-adjusted latency arrays ONCE. + + The adjustment ``max(latency - rtt, 0)`` is a per-record transform independent + of any aggregation window, so it is computed a single time over each full + source column (ms, full length, NaN preserved). The caller then aggregates + masked views of these arrays for the overall summary and each timeslice via + :func:`inject_network_adjusted_from_arrays` -- no redundant per-window + subtraction. ``rtt_ns`` is the run-level mean RTT (nanoseconds); caller + guarantees it is truthy. + """ + return { + adjusted_tag: np.maximum(store.numeric(source_tag) - rtt_ns, 0.0) / _NS_PER_MS + for adjusted_tag, source_tag in NETWORK_ADJUSTED_SOURCES.items() + } + + +def inject_network_adjusted_from_arrays( + adjusted_arrays: dict[str, NDArray[np.float64]], + results: dict[str, MetricResult], + rtt_ns: float, + mask: NDArray[np.bool_] | None = None, +) -> None: + """Aggregate precomputed adjusted arrays into ``results`` for one window. + + ``adjusted_arrays`` comes from :func:`compute_network_adjusted_arrays` (built + once per summarize). ``mask`` restricts to the window's records (a timeslice + bin or a phase-scoped export); None means all records. Also injects the + run-level ``network_rtt`` scalar. + """ + results[NetworkRttMetric.tag] = _network_rtt_result(rtt_ns) + for adjusted_tag, adjusted_ms in adjusted_arrays.items(): + values_ms = adjusted_ms if mask is None else adjusted_ms[mask] + valid_ms = values_ms[~np.isnan(values_ms)] + if valid_ms.size == 0: continue - adjusted_ms = np.maximum(valid_ns - rtt_ns, 0.0) / _NS_PER_MS results[adjusted_tag] = _array_to_metric_result( tag=adjusted_tag, header=_HEADER_BY_TAG[adjusted_tag], - values_ms=adjusted_ms, + values_ms=valid_ms, ) diff --git a/src/aiperf/metrics/types/accuracy_metrics.py b/src/aiperf/metrics/types/accuracy_metrics.py index 8f858b4bbd..84be54db7e 100644 --- a/src/aiperf/metrics/types/accuracy_metrics.py +++ b/src/aiperf/metrics/types/accuracy_metrics.py @@ -1,6 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from aiperf.accuracy.models import ( + ACCURACY_RECORD_CORRECT_KEY, + ACCURACY_RECORD_UNPARSED_KEY, +) from aiperf.common.enums import GenericMetricUnit, MetricConsoleGroup, MetricFlags from aiperf.common.exceptions import NoMetricValue from aiperf.common.models import ParsedResponseRecord @@ -9,15 +13,22 @@ class AccuracyCorrectSumMetric(BaseAggregateMetric[float]): - """Running sum of per-record accuracy.correct values (1.0 correct, 0.0 incorrect). + """Registration for the per-record ``accuracy_correct`` transport key. - AccuracyRecordProcessor writes this tag to MetricRecordDict for every record. - Registered here so MetricResultsProcessor can aggregate it without warnings. - AccuracyAccumulator and AccuracyConsoleExporter own display; this metric - uses console_group=NONE | INTERNAL so it does not appear in the standard table. + AccuracyRecordProcessor writes ``accuracy_correct`` (1.0 correct / 0.0 + incorrect) into every record's MetricRecordDict to hand the grade to + AccuracyResultsProcessor. MetricsAccumulator only ingests registered tags + (it warns and drops anything else), so this exists purely to register it. + + It is a transport key, NOT the accuracy display: ``INTERNAL`` + + ``console_group=NONE`` keep it out of the console table and file exports. + The user-facing accuracy summary is produced separately by + AccuracyResultsProcessor.summarize() under the ``accuracy.`` (dot) namespace, + which this ``accuracy_`` (underscore) tag deliberately stays out of so the + two can never collide. """ - tag = "accuracy.correct" + tag = ACCURACY_RECORD_CORRECT_KEY header = "Accuracy Correct" unit = GenericMetricUnit.RATIO flags = MetricFlags.INTERNAL @@ -27,9 +38,9 @@ class AccuracyCorrectSumMetric(BaseAggregateMetric[float]): def _parse_record( self, record: ParsedResponseRecord, record_metrics: MetricRecordDict ) -> float: - value = record_metrics.get("accuracy.correct") + value = record_metrics.get(ACCURACY_RECORD_CORRECT_KEY) if value is None: - raise NoMetricValue("accuracy.correct not in record_metrics") + raise NoMetricValue(f"{ACCURACY_RECORD_CORRECT_KEY} not in record_metrics") return float(value) def _aggregate_value(self, value: float) -> None: @@ -37,14 +48,18 @@ def _aggregate_value(self, value: float) -> None: class AccuracyUnparsedSumMetric(BaseAggregateMetric[float]): - """Running sum of per-record accuracy.unparsed values (1.0 unparsed, 0.0 conforming). + """Registration for the per-record ``accuracy_unparsed`` transport key. - AccuracyRecordProcessor writes this tag when the model output required the - regex fallback (e.g. 'The answer is B.' instead of 'B'). - Uses console_group=NONE | INTERNAL so it does not appear in the standard table. + Same role as AccuracyCorrectSumMetric: registers the per-record + ``accuracy_unparsed`` (1.0 when the grader could not cleanly extract the + answer) transport key so MetricsAccumulator accepts it. Transport only -- + ``INTERNAL`` + ``console_group=NONE``; the displayed unparsed counts come + from AccuracyResultsProcessor.summarize() in the ``accuracy.`` namespace. + Registering this under the dot tag ``accuracy.unparsed`` used to collide + with that summary tag and drop the overall unparsed count. """ - tag = "accuracy.unparsed" + tag = ACCURACY_RECORD_UNPARSED_KEY header = "Accuracy Unparsed" unit = GenericMetricUnit.RATIO flags = MetricFlags.INTERNAL @@ -54,9 +69,9 @@ class AccuracyUnparsedSumMetric(BaseAggregateMetric[float]): def _parse_record( self, record: ParsedResponseRecord, record_metrics: MetricRecordDict ) -> float: - value = record_metrics.get("accuracy.unparsed") + value = record_metrics.get(ACCURACY_RECORD_UNPARSED_KEY) if value is None: - raise NoMetricValue("accuracy.unparsed not in record_metrics") + raise NoMetricValue(f"{ACCURACY_RECORD_UNPARSED_KEY} not in record_metrics") return float(value) def _aggregate_value(self, value: float) -> None: diff --git a/src/aiperf/metrics/types/network_adjusted_metrics.py b/src/aiperf/metrics/types/network_adjusted_metrics.py index e133b7d78c..70847d5708 100644 --- a/src/aiperf/metrics/types/network_adjusted_metrics.py +++ b/src/aiperf/metrics/types/network_adjusted_metrics.py @@ -8,10 +8,10 @@ comparable. The raw metrics are always preserved. These metrics are NOT computed per-record. The RTT is a run-level scalar that is only -known at the end of the run, so :class:`aiperf.post_processors.metric_results_processor` -injects the shifted distributions after aggregation (subtracting a constant shifts every -percentile and the mean by that constant and leaves the standard deviation unchanged). -``_derive_value`` therefore always defers. +known at the end of the run, so MetricsAccumulator injects the shifted distributions +after aggregation (subtracting a constant shifts every percentile and the mean by that +constant and leaves the standard deviation unchanged). ``_derive_value`` therefore +always defers. Inter-token / inter-chunk latencies are intentionally NOT adjusted: the same RTT cancels in ``(request_latency - ttft)``, so those metrics are already network-invariant. diff --git a/src/aiperf/metrics/types/power_efficiency_metrics.py b/src/aiperf/metrics/types/power_efficiency_metrics.py index cfb1395cca..e970327796 100644 --- a/src/aiperf/metrics/types/power_efficiency_metrics.py +++ b/src/aiperf/metrics/types/power_efficiency_metrics.py @@ -20,7 +20,7 @@ class TotalGpuPowerMetric(BaseDerivedMetric[float]): Invariant: externally injected by `GPUTelemetryAccumulator.compute_efficiency_metrics` from gpu_power_usage scrapes. `_derive_value` is intentionally non-functional; - `MetricResultsProcessor.update_derived_metrics` is expected to catch + `MetricsAccumulator._resolve_derived_metrics` is expected to catch NoMetricValue and skip the tag during its derivation walk. """ @@ -36,7 +36,7 @@ def _derive_value(self, metric_results: MetricResultsDict) -> NoReturn: "is externally injected by " "GPUTelemetryAccumulator.compute_efficiency_metrics. If this exception " "surfaces, the derivation walk is missing its NoMetricValue handler " - "(see MetricResultsProcessor.update_derived_metrics)." + "(see MetricsAccumulator._resolve_derived_metrics)." ) @@ -46,7 +46,7 @@ class TotalGpuEnergyMetric(BaseDerivedMetric[float]): Invariant: externally injected by `GPUTelemetryAccumulator.compute_efficiency_metrics` from energy_consumption counter deltas. `_derive_value` is intentionally - non-functional; `MetricResultsProcessor.update_derived_metrics` is + non-functional; `MetricsAccumulator._resolve_derived_metrics` is expected to catch NoMetricValue and skip the tag during its derivation walk. """ @@ -63,7 +63,7 @@ def _derive_value(self, metric_results: MetricResultsDict) -> NoReturn: "is externally injected by " "GPUTelemetryAccumulator.compute_efficiency_metrics. If this exception " "surfaces, the derivation walk is missing its NoMetricValue handler " - "(see MetricResultsProcessor.update_derived_metrics)." + "(see MetricsAccumulator._resolve_derived_metrics)." ) @@ -74,7 +74,7 @@ class OutputTokensPerJouleMetric(BaseDerivedMetric[float]): `GPUTelemetryAccumulator.compute_efficiency_metrics` as `total_output_tokens / total_gpu_energy`. `_derive_value` is intentionally non-functional; - `MetricResultsProcessor.update_derived_metrics` is expected to catch + `MetricsAccumulator._resolve_derived_metrics` is expected to catch NoMetricValue and skip the tag during its derivation walk. """ @@ -90,7 +90,7 @@ def _derive_value(self, metric_results: MetricResultsDict) -> NoReturn: "metric is externally injected by " "GPUTelemetryAccumulator.compute_efficiency_metrics. If this exception " "surfaces, the derivation walk is missing its NoMetricValue handler " - "(see MetricResultsProcessor.update_derived_metrics)." + "(see MetricsAccumulator._resolve_derived_metrics)." ) @@ -101,7 +101,7 @@ class EnergyPerUserMetric(BaseDerivedMetric[float]): `GPUTelemetryAccumulator.compute_efficiency_metrics` as `total_gpu_energy / profiling_phase.concurrency`. `_derive_value` is intentionally non-functional; - `MetricResultsProcessor.update_derived_metrics` is expected to catch + `MetricsAccumulator._resolve_derived_metrics` is expected to catch NoMetricValue and skip the tag during its derivation walk. Omitted when concurrency is unset (e.g. pure request-rate runs) or zero. """ @@ -118,5 +118,5 @@ def _derive_value(self, metric_results: MetricResultsDict) -> NoReturn: "is externally injected by " "GPUTelemetryAccumulator.compute_efficiency_metrics. If this exception " "surfaces, the derivation walk is missing its NoMetricValue handler " - "(see MetricResultsProcessor.update_derived_metrics)." + "(see MetricsAccumulator._resolve_derived_metrics)." ) diff --git a/src/aiperf/network_latency/accumulator.py b/src/aiperf/network_latency/accumulator.py index af748d181d..409e200f70 100644 --- a/src/aiperf/network_latency/accumulator.py +++ b/src/aiperf/network_latency/accumulator.py @@ -43,10 +43,10 @@ def _percentile_stats(rtts_ns: list[int]) -> dict[str, float | None]: class NetworkLatencyAccumulator: """Accumulates RTT probe samples and computes per-target + aggregate stats. - Instantiated directly by the RecordsManager (not a plugin results processor) - because the aggregate ``mean_ns`` must be delivered to every - MetricResultsProcessor via ``set_network_rtt_ns`` before ``summarize()`` is - called, rather than flowing through the standard summarize pipeline. + Instantiated directly by the RecordsManager because the aggregate + ``mean_ns`` must be delivered to MetricsAccumulator via ``set_network_rtt_ns`` + before ``summarize()`` is called, rather than flowing through the standard + summarize pipeline. """ def __init__(self, benchmark_id: str | None = None) -> None: diff --git a/src/aiperf/network_latency/jsonl_writer.py b/src/aiperf/network_latency/jsonl_writer.py index c5c108a5a0..54c709a9a7 100644 --- a/src/aiperf/network_latency/jsonl_writer.py +++ b/src/aiperf/network_latency/jsonl_writer.py @@ -50,11 +50,13 @@ def __init__( self.info(f"Network latency JSONL export enabled: {self.output_file}") - async def process_network_latency_sample( - self, sample: NetworkLatencySample - ) -> None: + async def process_record(self, record: NetworkLatencySample) -> None: """Write a single probe sample to the JSONL artifact.""" - await self.buffered_write(sample) + await self.buffered_write(record) + + async def finalize(self) -> None: + """Flush buffered probe samples before final artifact publication.""" + await self.flush_buffer() async def summarize(self) -> list[MetricResult]: """Summarize result. Not used for this processor.""" diff --git a/src/aiperf/network_latency/protocols.py b/src/aiperf/network_latency/protocols.py deleted file mode 100644 index 23260c374b..0000000000 --- a/src/aiperf/network_latency/protocols.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -from typing import TYPE_CHECKING, Protocol, runtime_checkable - -if TYPE_CHECKING: - from aiperf.common.models import MetricResult, NetworkLatencySample - - -@runtime_checkable -class NetworkLatencyProcessorProtocol(Protocol): - """Protocol for results processors that consume network latency RTT samples. - - Separate from ResultsProcessorProtocol because RTT probe samples are - structurally distinct from inference metric records. - """ - - async def process_network_latency_sample( - self, sample: NetworkLatencySample - ) -> None: - """Process a single TCP-handshake RTT probe sample. - - Args: - sample: NetworkLatencySample with the probe result (success or failure) - """ - ... - - async def summarize(self) -> list[MetricResult]: ... diff --git a/src/aiperf/orchestrator/search_planner/_pooled_percentile.py b/src/aiperf/orchestrator/search_planner/_pooled_percentile.py index e1cb765057..5d498a839a 100644 --- a/src/aiperf/orchestrator/search_planner/_pooled_percentile.py +++ b/src/aiperf/orchestrator/search_planner/_pooled_percentile.py @@ -9,7 +9,7 @@ correct statistic; mean-of-percentiles is a heuristic. This helper walks each ``RunResult.artifacts_path / "profile_export.jsonl"`` -file -- written by :mod:`aiperf.post_processors.record_export_results_processor` +file -- written by :mod:`aiperf.post_processors.record_export_jsonl_writer` when ``--export-level records`` (or ``raw``) is set -- accumulates raw metric samples into a single bag, and returns ``np.percentile(bag, pct)``. diff --git a/src/aiperf/plugin/categories.yaml b/src/aiperf/plugin/categories.yaml index b7ae106e73..fc3a15b9d0 100644 --- a/src/aiperf/plugin/categories.yaml +++ b/src/aiperf/plugin/categories.yaml @@ -152,33 +152,6 @@ record_processor: First stage of metrics pipeline, handling per-record computations. One-to-many mapping: multiple processors can be loaded simultaneously. -results_processor: - protocol: aiperf.post_processors.base_metrics_processor:BaseMetricsProcessor - enum: ResultsProcessorType - description: | - Results processors aggregate results from record processors and compute derived metrics. - Final stage of metrics pipeline for aggregated statistics and summaries. - One-to-many mapping: multiple processors can be loaded simultaneously. - -gpu_telemetry_processor: - protocol: aiperf.post_processors.base_metrics_processor:BaseMetricsProcessor - enum: GPUTelemetryProcessorType - description: | - GPU telemetry processors aggregate and export GPU telemetry records locally - within GPUTelemetryManager. Side-channel pipeline that keeps telemetry - records in their origin process rather than routing through RecordsManager. - One-to-many mapping: multiple processors can be loaded simultaneously. - -server_metrics_processor: - protocol: aiperf.post_processors.base_metrics_processor:BaseMetricsProcessor - enum: ServerMetricsProcessorType - description: | - Server metrics processors aggregate and export Prometheus server metrics - records locally within ServerMetricsManager. Side-channel pipeline that - keeps server metrics records in their origin process rather than routing - through RecordsManager. - One-to-many mapping: multiple processors can be loaded simultaneously. - accumulator: protocol: aiperf.common.accumulator_protocols:AccumulatorProtocol metadata_class: aiperf.plugin.schema.schemas:RecordRoutingMetadata diff --git a/src/aiperf/plugin/enums.py b/src/aiperf/plugin/enums.py index f578918a58..38e2daa652 100644 --- a/src/aiperf/plugin/enums.py +++ b/src/aiperf/plugin/enums.py @@ -12,7 +12,7 @@ from aiperf.plugin import plugins from aiperf.plugin.extensible_enums import create_enum -__all__ = ["APIRouterType", "APIRouterTypeStr", "AccumulatorType", "AccumulatorTypeStr", "AccuracyBenchmarkType", "AccuracyBenchmarkTypeStr", "AccuracyGraderType", "AccuracyGraderTypeStr", "ArrivalPattern", "ArrivalPatternStr", "CommClientType", "CommClientTypeStr", "CommunicationBackend", "CommunicationBackendStr", "ComposerType", "ComposerTypeStr", "ConsoleExporterType", "ConsoleExporterTypeStr", "ConvergenceCriterionType", "ConvergenceCriterionTypeStr", "CustomDatasetType", "CustomDatasetTypeStr", "DataExporterType", "DataExporterTypeStr", "DatasetBackingStoreType", "DatasetBackingStoreTypeStr", "DatasetClientStoreType", "DatasetClientStoreTypeStr", "DatasetFormat", "DatasetFormatStr", "DatasetSamplingStrategy", "DatasetSamplingStrategyStr", "EndpointType", "EndpointTypeStr", "GPUTelemetryCollectorType", "GPUTelemetryCollectorTypeStr", "GPUTelemetryProcessorType", "GPUTelemetryProcessorTypeStr", "PhaseType", "PhaseTypeStr", "PlotType", "PlotTypeStr", "PluginType", "PluginTypeStr", "PublicDatasetType", "PublicDatasetTypeStr", "RampType", "RampTypeStr", "RecordProcessorType", "RecordProcessorTypeStr", "ResultsProcessorType", "ResultsProcessorTypeStr", "SearchPlannerType", "SearchPlannerTypeStr", "SearchRecipePostProcessType", "SearchRecipePostProcessTypeStr", "SearchRecipeType", "SearchRecipeTypeStr", "ServerMetricsProcessorType", "ServerMetricsProcessorTypeStr", "ServiceRunType", "ServiceRunTypeStr", "ServiceType", "ServiceTypeStr", "StreamExporterType", "StreamExporterTypeStr", "TimingMode", "TimingModeStr", "TransportType", "TransportTypeStr", "UIType", "UITypeStr", "URLSelectionStrategy", "URLSelectionStrategyStr", "ZMQProxyType", "ZMQProxyTypeStr"] +__all__ = ["APIRouterType", "APIRouterTypeStr", "AccumulatorType", "AccumulatorTypeStr", "AccuracyBenchmarkType", "AccuracyBenchmarkTypeStr", "AccuracyGraderType", "AccuracyGraderTypeStr", "ArrivalPattern", "ArrivalPatternStr", "CommClientType", "CommClientTypeStr", "CommunicationBackend", "CommunicationBackendStr", "ComposerType", "ComposerTypeStr", "ConsoleExporterType", "ConsoleExporterTypeStr", "ConvergenceCriterionType", "ConvergenceCriterionTypeStr", "CustomDatasetType", "CustomDatasetTypeStr", "DataExporterType", "DataExporterTypeStr", "DatasetBackingStoreType", "DatasetBackingStoreTypeStr", "DatasetClientStoreType", "DatasetClientStoreTypeStr", "DatasetFormat", "DatasetFormatStr", "DatasetSamplingStrategy", "DatasetSamplingStrategyStr", "EndpointType", "EndpointTypeStr", "GPUTelemetryCollectorType", "GPUTelemetryCollectorTypeStr", "PhaseType", "PhaseTypeStr", "PlotType", "PlotTypeStr", "PluginType", "PluginTypeStr", "PublicDatasetType", "PublicDatasetTypeStr", "RampType", "RampTypeStr", "RecordProcessorType", "RecordProcessorTypeStr", "SearchPlannerType", "SearchPlannerTypeStr", "SearchRecipePostProcessType", "SearchRecipePostProcessTypeStr", "SearchRecipeType", "SearchRecipeTypeStr", "ServiceRunType", "ServiceRunTypeStr", "ServiceType", "ServiceTypeStr", "StreamExporterType", "StreamExporterTypeStr", "TimingMode", "TimingModeStr", "TransportType", "TransportTypeStr", "UIType", "UITypeStr", "URLSelectionStrategy", "URLSelectionStrategyStr", "ZMQProxyType", "ZMQProxyTypeStr"] # Plugin Protocol Categories if TYPE_CHECKING: @@ -77,25 +77,13 @@ RecordProcessorType = plugins.create_enum(PluginType.RECORD_PROCESSOR, "RecordProcessorType", module=__name__) """Dynamic enum for record processor. Example: RecordProcessorType.ACCURACY_RECORD, RecordProcessorType.OUTPUTS_JSON, RecordProcessorType.RAW_RECORD_WRITER""" -ResultsProcessorTypeStr: TypeAlias = str -ResultsProcessorType = plugins.create_enum(PluginType.RESULTS_PROCESSOR, "ResultsProcessorType", module=__name__) -"""Dynamic enum for results processor. Example: ResultsProcessorType.ACCURACY_RESULTS, ResultsProcessorType.NETWORK_LATENCY_JSONL_WRITER, ResultsProcessorType.TIMESLICE""" - -GPUTelemetryProcessorTypeStr: TypeAlias = str -GPUTelemetryProcessorType = plugins.create_enum(PluginType.GPU_TELEMETRY_PROCESSOR, "GPUTelemetryProcessorType", module=__name__) -"""Dynamic enum for gpu telemetry processor. Example: GPUTelemetryProcessorType.GPU_TELEMETRY_ACCUMULATOR""" - -ServerMetricsProcessorTypeStr: TypeAlias = str -ServerMetricsProcessorType = plugins.create_enum(PluginType.SERVER_METRICS_PROCESSOR, "ServerMetricsProcessorType", module=__name__) -"""Dynamic enum for server metrics processor. Example: ServerMetricsProcessorType.SERVER_METRICS_ACCUMULATOR""" - AccumulatorTypeStr: TypeAlias = str AccumulatorType = plugins.create_enum(PluginType.ACCUMULATOR, "AccumulatorType", module=__name__) -"""Dynamic enum for accumulator. Example: AccumulatorType.GPU_TELEMETRY, AccumulatorType.METRIC_RESULTS, AccumulatorType.SERVER_METRICS""" +"""Dynamic enum for accumulator. Example: AccumulatorType.ACCURACY_RESULTS, AccumulatorType.METRIC_RESULTS, AccumulatorType.SERVER_METRICS""" StreamExporterTypeStr: TypeAlias = str StreamExporterType = plugins.create_enum(PluginType.STREAM_EXPORTER, "StreamExporterType", module=__name__) -"""Dynamic enum for stream exporter. Example: StreamExporterType.GPU_TELEMETRY_JSONL_WRITER, StreamExporterType.RECORD_EXPORT, StreamExporterType.SERVER_METRICS_JSONL_WRITER""" +"""Dynamic enum for stream exporter. Example: StreamExporterType.GPU_TELEMETRY_JSONL_WRITER, StreamExporterType.OTEL_METRICS_STREAMER, StreamExporterType.SERVER_METRICS_JSONL_WRITER""" AccuracyGraderTypeStr: TypeAlias = str AccuracyGraderType = plugins.create_enum(PluginType.ACCURACY_GRADER, "AccuracyGraderType", module=__name__) diff --git a/src/aiperf/plugin/plugins.py b/src/aiperf/plugin/plugins.py index e337956a96..f108eddbd6 100644 --- a/src/aiperf/plugin/plugins.py +++ b/src/aiperf/plugin/plugins.py @@ -939,8 +939,7 @@ def _load_package_metadata( from aiperf.orchestrator.convergence.base import ConvergenceCriterion from aiperf.orchestrator.search_planner.base import SearchPlanner from aiperf.plot.core.plot_type_handlers import PlotTypeHandlerProtocol - from aiperf.plugin.enums import APIRouterType, AccumulatorType, AccuracyBenchmarkType, AccuracyGraderType, ArrivalPattern, CommClientType, CommunicationBackend, ComposerType, ConsoleExporterType, ConvergenceCriterionType, CustomDatasetType, DataExporterType, DatasetBackingStoreType, DatasetClientStoreType, DatasetSamplingStrategy, EndpointType, GPUTelemetryCollectorType, GPUTelemetryProcessorType, PlotType, PluginType, PluginTypeStr, PublicDatasetType, RampType, RecordProcessorType, ResultsProcessorType, SearchPlannerType, SearchRecipePostProcessType, SearchRecipeType, ServerMetricsProcessorType, ServiceRunType, ServiceType, StreamExporterType, TimingMode, TransportType, UIType, URLSelectionStrategy, ZMQProxyType - from aiperf.post_processors.base_metrics_processor import BaseMetricsProcessor + from aiperf.plugin.enums import APIRouterType, AccumulatorType, AccuracyBenchmarkType, AccuracyGraderType, ArrivalPattern, CommClientType, CommunicationBackend, ComposerType, ConsoleExporterType, ConvergenceCriterionType, CustomDatasetType, DataExporterType, DatasetBackingStoreType, DatasetClientStoreType, DatasetSamplingStrategy, EndpointType, GPUTelemetryCollectorType, PlotType, PluginType, PluginTypeStr, PublicDatasetType, RampType, RecordProcessorType, SearchPlannerType, SearchRecipePostProcessType, SearchRecipeType, ServiceRunType, ServiceType, StreamExporterType, TimingMode, TransportType, UIType, URLSelectionStrategy, ZMQProxyType from aiperf.post_processors.protocols import RecordProcessorProtocol from aiperf.search_recipes._base import SearchRecipe from aiperf.search_recipes.post_process import PostProcessHandler @@ -1007,18 +1006,6 @@ def get_class(category: Literal[PluginType.RECORD_PROCESSOR, "record_processor"] @overload def iter_all(category: Literal[PluginType.RECORD_PROCESSOR, "record_processor"]) -> Iterator[tuple[PluginEntry, type[RecordProcessorProtocol]]]: ... @overload - def get_class(category: Literal[PluginType.RESULTS_PROCESSOR, "results_processor"], name_or_class_path: ResultsProcessorType | str) -> type[BaseMetricsProcessor]: ... - @overload - def iter_all(category: Literal[PluginType.RESULTS_PROCESSOR, "results_processor"]) -> Iterator[tuple[PluginEntry, type[BaseMetricsProcessor]]]: ... - @overload - def get_class(category: Literal[PluginType.GPU_TELEMETRY_PROCESSOR, "gpu_telemetry_processor"], name_or_class_path: GPUTelemetryProcessorType | str) -> type[BaseMetricsProcessor]: ... - @overload - def iter_all(category: Literal[PluginType.GPU_TELEMETRY_PROCESSOR, "gpu_telemetry_processor"]) -> Iterator[tuple[PluginEntry, type[BaseMetricsProcessor]]]: ... - @overload - def get_class(category: Literal[PluginType.SERVER_METRICS_PROCESSOR, "server_metrics_processor"], name_or_class_path: ServerMetricsProcessorType | str) -> type[BaseMetricsProcessor]: ... - @overload - def iter_all(category: Literal[PluginType.SERVER_METRICS_PROCESSOR, "server_metrics_processor"]) -> Iterator[tuple[PluginEntry, type[BaseMetricsProcessor]]]: ... - @overload def get_class(category: Literal[PluginType.ACCUMULATOR, "accumulator"], name_or_class_path: AccumulatorType | str) -> type[AccumulatorProtocol]: ... @overload def iter_all(category: Literal[PluginType.ACCUMULATOR, "accumulator"]) -> Iterator[tuple[PluginEntry, type[AccumulatorProtocol]]]: ... diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index 7715db01da..31eb4da444 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -77,9 +77,8 @@ service: records_manager: class: aiperf.records.records_manager:RecordsManager description: | - Record aggregation and results management service. Aggregates metrics from all - record processors, manages results processors, and coordinates final results - export. + Record aggregation and results management service. Routes records to + accumulators and stream exporters, and coordinates final results export. metadata: required: true auto_start: true @@ -976,67 +975,12 @@ record_processor: Outputs JSON record processor that captures model response text per request. Enabled when --export-outputs-json is set. Writes per-processor fragment files. -# ============================================================================= -# Results Processors -# ============================================================================= -# Results processors aggregate results from record processors and compute derived metrics. -# One-to-many mapping: multiple processors can be loaded simultaneously. -# ============================================================================= -results_processor: - gpu_telemetry_accumulator: - class: aiperf.gpu_telemetry.accumulator:GPUTelemetryAccumulator - description: | - GPU telemetry accumulator that aggregates GPU telemetry records and computes - metrics in a hierarchical structure. Loaded when telemetry is enabled. - - metric_results: - class: aiperf.post_processors.metric_results_processor:MetricResultsProcessor - description: | - Results processor that computes metrics from MetricType.DERIVED and - aggregates results from all record processors. Final stage of metrics - pipeline. Always loaded. - - network_latency_jsonl_writer: - class: aiperf.network_latency.jsonl_writer:NetworkLatencyJSONLWriter - description: | - Network latency JSONL writer that exports per-sample TCP-handshake RTT - probes to a JSONL file. Enabled when network latency calibration is - actively probing (no mean_ms). - - otel_metrics_streamer: - class: aiperf.post_processors.otel_metrics_results_processor:OTelMetricsResultsProcessor - description: | - Streams per-record metrics to a user-provided OpenTelemetry collector - with periodic export. - Enabled when --otel-url is configured. - - server_metrics_accumulator: - class: aiperf.server_metrics.accumulator:ServerMetricsAccumulator - description: | - Server metrics accumulator that aggregates Prometheus server metrics records - and computes summary statistics. Supports Gauge, Counter, and Histogram metrics. - - timeslice: - class: aiperf.post_processors.timeslice_metric_results_processor:TimesliceMetricResultsProcessor - description: | - Timeslice results processor that computes metrics for user-configurable - time slices, enabling time-series analysis of benchmark performance. - Enabled when timeslice config is set. - - accuracy_results: - class: aiperf.accuracy.accuracy_results_processor:AccuracyResultsProcessor - description: | - Accuracy results processor that aggregates grading results and computes - summary accuracy metrics. Self-disables when accuracy mode is off. - # ============================================================================= # Accumulators # ============================================================================= # Accumulators ingest records, support time-range queries, and produce summaries. -# Primary data stores in the records pipeline (RecordsManager). The -# MetricsAccumulator is the summary engine for the records pipeline; the legacy -# results_processor:metric_results (MetricResultsProcessor) is gated OFF when -# the accumulator path is active to avoid double-compute. +# Primary data stores in the records pipeline (RecordsManager). Each accumulator +# declares the record types it consumes in metadata.record_types. # One-to-many mapping: multiple accumulators loaded simultaneously. # ============================================================================= accumulator: @@ -1065,6 +1009,14 @@ accumulator: metadata: record_types: [server_metrics] + accuracy_results: + class: aiperf.accuracy.accuracy_results_processor:AccuracyResultsProcessor + description: | + Accuracy accumulator that ingests graded metric records and computes summary + accuracy metrics. Self-disables when accuracy mode is off. + metadata: + record_types: [metric_records] + # ============================================================================= # Stream Exporters # ============================================================================= @@ -1082,6 +1034,24 @@ stream_exporter: metadata: record_types: [gpu_telemetry] + network_latency_jsonl_writer: + class: aiperf.network_latency.jsonl_writer:NetworkLatencyJSONLWriter + description: | + Network latency JSONL writer that exports per-sample TCP-handshake RTT + probes to a JSONL file. Enabled when network latency calibration is + actively probing (no mean_ms). + metadata: + record_types: [network_latency] + + otel_metrics_streamer: + class: aiperf.post_processors.otel_metrics_results_processor:OTelMetricsResultsProcessor + description: | + Streams per-record metrics and timing snapshots to a user-provided + OpenTelemetry collector with periodic export. Enabled when --otel-url or + MLflow live streaming is configured. + metadata: + record_types: [metric_records, credit_phase_stats] + record_export: class: aiperf.post_processors.record_export_jsonl_writer:RecordExportJSONLWriter description: | @@ -1098,32 +1068,6 @@ stream_exporter: metadata: record_types: [server_metrics] -# ============================================================================= -# GPU Telemetry Processors -# ============================================================================= -# GPU telemetry processors aggregate and export GPU telemetry records locally -# within GPUTelemetryManager. One-to-many mapping. -# ============================================================================= -gpu_telemetry_processor: - gpu_telemetry_accumulator: - class: aiperf.gpu_telemetry.accumulator:GPUTelemetryAccumulator - description: | - GPU telemetry accumulator that aggregates GPU telemetry records and computes - metrics in a hierarchical structure. Loaded when telemetry is enabled. - -# ============================================================================= -# Server Metrics Processors -# ============================================================================= -# Server metrics processors aggregate and export Prometheus server metrics -# records locally within ServerMetricsManager. One-to-many mapping. -# ============================================================================= -server_metrics_processor: - server_metrics_accumulator: - class: aiperf.server_metrics.accumulator:ServerMetricsAccumulator - description: | - Server metrics accumulator that aggregates Prometheus server metrics records - and computes summary statistics. Supports Gauge, Counter, and Histogram metrics. - # ============================================================================= # Artifact Publishers # ============================================================================= diff --git a/src/aiperf/plugin/schema/plugins.schema.json b/src/aiperf/plugin/schema/plugins.schema.json index 227be5d26c..a243deb897 100644 --- a/src/aiperf/plugin/schema/plugins.schema.json +++ b/src/aiperf/plugin/schema/plugins.schema.json @@ -114,30 +114,6 @@ "$ref": "#/$defs/RecordProcessorPlugin" } }, - "results_processor": { - "title": "Results Processor Plugins", - "type": "object", - "description": "Results processors aggregate results from record processors and compute derived metrics.\nFinal stage of metrics pipeline for aggregated statistics and summaries.\nOne-to-many mapping: multiple processors can be loaded simultaneously.", - "additionalProperties": { - "$ref": "#/$defs/ResultsProcessorPlugin" - } - }, - "gpu_telemetry_processor": { - "title": "GPU Telemetry Processor Plugins", - "type": "object", - "description": "GPU telemetry processors aggregate and export GPU telemetry records locally\nwithin GPUTelemetryManager. Side-channel pipeline that keeps telemetry\nrecords in their origin process rather than routing through RecordsManager.\nOne-to-many mapping: multiple processors can be loaded simultaneously.", - "additionalProperties": { - "$ref": "#/$defs/GpuTelemetryProcessorPlugin" - } - }, - "server_metrics_processor": { - "title": "Server Metrics Processor Plugins", - "type": "object", - "description": "Server metrics processors aggregate and export Prometheus server metrics\nrecords locally within ServerMetricsManager. Side-channel pipeline that\nkeeps server metrics records in their origin process rather than routing\nthrough RecordsManager.\nOne-to-many mapping: multiple processors can be loaded simultaneously.", - "additionalProperties": { - "$ref": "#/$defs/ServerMetricsProcessorPlugin" - } - }, "accumulator": { "title": "Accumulator Plugins", "type": "object", @@ -1107,129 +1083,6 @@ "title": "Record Processor Plugin", "description": "Record processors stream records and compute metrics in a distributed manner.\nFirst stage of metrics pipeline, handling per-record computations.\nOne-to-many mapping: multiple processors can be loaded simultaneously." }, - "ResultsProcessorPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Results Processor Plugin", - "description": "Results processors aggregate results from record processors and compute derived metrics.\nFinal stage of metrics pipeline for aggregated statistics and summaries.\nOne-to-many mapping: multiple processors can be loaded simultaneously." - }, - "GpuTelemetryProcessorPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "GPU Telemetry Processor Plugin", - "description": "GPU telemetry processors aggregate and export GPU telemetry records locally\nwithin GPUTelemetryManager. Side-channel pipeline that keeps telemetry\nrecords in their origin process rather than routing through RecordsManager.\nOne-to-many mapping: multiple processors can be loaded simultaneously." - }, - "ServerMetricsProcessorPlugin": { - "type": "object", - "properties": { - "class": { - "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", - "title": "Class", - "type": "string" - }, - "description": { - "default": "", - "description": "Brief explanation of what this plugin type does and when to use it.", - "title": "Description", - "type": "string" - }, - "priority": { - "default": 0, - "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", - "title": "Priority", - "type": "integer" - }, - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Category-specific configuration for this plugin type. The allowed fields depend on the category's metadata_class in categories.yaml.", - "title": "Metadata" - } - }, - "required": [ - "class" - ], - "title": "Server Metrics Processor Plugin", - "description": "Server metrics processors aggregate and export Prometheus server metrics\nrecords locally within ServerMetricsManager. Side-channel pipeline that\nkeeps server metrics records in their origin process rather than routing\nthrough RecordsManager.\nOne-to-many mapping: multiple processors can be loaded simultaneously." - }, "AccumulatorPlugin": { "type": "object", "properties": { diff --git a/src/aiperf/post_processors/metric_results_processor.py b/src/aiperf/post_processors/metric_results_processor.py deleted file mode 100644 index 941084c832..0000000000 --- a/src/aiperf/post_processors/metric_results_processor.py +++ /dev/null @@ -1,275 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - -from collections.abc import Callable -from typing import TYPE_CHECKING, Any - -import numpy as np - -from aiperf.common.enums import ( - MetricDictValueTypeT, - MetricFlags, - MetricType, - MetricValueTypeT, -) -from aiperf.common.environment import Environment -from aiperf.common.exceptions import NoMetricValue -from aiperf.common.messages.inference_messages import MetricRecordsData -from aiperf.common.models import MetricResult -from aiperf.common.types import MetricTagT -from aiperf.metrics import BaseAggregateMetric -from aiperf.metrics.base_metric import BaseMetric -from aiperf.metrics.display_units import to_display_unit -from aiperf.metrics.list_metric_aggregation import TDigestListMetricAggregator -from aiperf.metrics.metric_dicts import MetricAggregator, MetricArray, MetricResultsDict -from aiperf.metrics.metric_registry import MetricRegistry -from aiperf.metrics.types.network_adjusted_metrics import ( - NETWORK_ADJUSTED_SOURCES, - NetworkRttMetric, -) -from aiperf.post_processors.base_metrics_processor import BaseMetricsProcessor - -if TYPE_CHECKING: - from aiperf.config.resolution.plan import BenchmarkRun - - -class MetricResultsProcessor(BaseMetricsProcessor): - """Processor for metric results. - - This is the final stage of the metrics processing pipeline, and is done is a unified manner by the RecordsManager. - It is responsible for processing the results and returning them to the RecordsManager, as well as summarizing the results. - """ - - def __init__(self, run: BenchmarkRun, **kwargs: Any): - super().__init__(run=run, **kwargs) - # For derived metrics, we don't care about splitting up the error metrics - # Note: _setup_metrics returns metrics in dependency order, which includes - # non-derived dependencies. We filter to only include actual derived metrics. - self.derive_funcs: dict[ - MetricTagT, Callable[[MetricResultsDict], MetricValueTypeT] - ] = { - metric.tag: metric.derive_value # type: ignore - for metric in self._setup_metrics(MetricType.DERIVED) - if metric.type == MetricType.DERIVED - } - - # Create the results dict, which will be used to store the results of non-derived metrics, - # and then be updated with the derived metrics. - self._results: MetricResultsDict = MetricResultsDict() - - # Run-level mean network RTT (ns) to subtract from request-start-anchored - # latency metrics. Set by the RecordsManager before summarize() when network - # latency calibration is enabled; None means no adjustment is applied. - self._network_rtt_ns: float | None = None - - # Get all of the metric classes. - _all_metric_classes: list[type[BaseMetric]] = MetricRegistry.all_classes() - - # Pre-cache the types for the metrics. - self._tags_to_types: dict[MetricTagT, MetricType] = { - metric.tag: metric.type for metric in _all_metric_classes - } - - # Set up aggregate metric objects - self._instances_map: dict[MetricTagT, BaseMetric] = { - tag: MetricRegistry.get_class(tag)() for tag in MetricRegistry.all_tags() - } - - # Pre-cache the aggregate functions for the aggregate metrics. - self._tags_to_aggregate_funcs: dict[ - MetricTagT, Callable[[MetricResultsDict], MetricValueTypeT] - ] = { - metric.tag: MetricRegistry.get_instance(metric.tag).aggregate_value # type: ignore - for metric in _all_metric_classes - if metric.type == MetricType.AGGREGATE - } - - async def process_result(self, record_data: MetricRecordsData) -> None: - """Process a result from the metric record processor.""" - if self.is_trace_enabled: - self.trace(f"Processing incoming metrics: {record_data.metrics}") - - # Get the appropriate results dict and instances map once to avoid multiple calls - request_start_ns = record_data.metadata.request_start_ns - instances_map = await self.get_instances_map(request_start_ns) - results_dict = await self.get_results(request_start_ns) - - for tag, value in record_data.metrics.items(): - try: - metric_type = self._tags_to_types[tag] - if metric_type == MetricType.RECORD: - if tag not in results_dict: - # The metric class shape doesn't change mid-run, so the - # storage type can be picked at first-touch. List values - # go to the bounded t-digest aggregator (e.g. - # inter_chunk_latency, which would otherwise blow past - # pod RAM at ramp scale); scalar values stay in - # MetricArray. - results_dict[tag] = ( - TDigestListMetricAggregator() - if isinstance(value, list) - else MetricArray() - ) - if isinstance(value, list): - results_dict[tag].extend(value) - else: - results_dict[tag].append(value) - - elif metric_type == MetricType.AGGREGATE: - metric: BaseAggregateMetric = instances_map[tag] # type: ignore - metric.aggregate_value(value) - results_dict[tag] = metric.current_value - - else: - raise ValueError(f"Metric '{tag}' is not a valid metric type") - except NoMetricValue as e: - self.trace( - lambda tag=tag, e=e: f"No metric value for metric '{tag}': {e!r}" - ) - except Exception as e: - self.warning(f"Error processing metric '{tag}': {e!r}") - - if self.is_trace_enabled: - self.trace(f"Results after processing incoming metrics: {results_dict}") - - async def get_instances_map( - self, request_start_ns: int | None = None - ) -> dict[MetricTagT, BaseMetric]: - """Get the appropriate instances map based on mode. - - In non-timeslice mode, returns the single shared instances map. - Subclasses can override to provide timeslice-specific behavior. - """ - return self._instances_map - - async def get_results( - self, request_start_ns: int | None = None - ) -> MetricResultsDict: - """Get the appropriate results dictionary based on mode. - - In non-timeslice mode, returns the single shared results dict. - Subclasses can override to provide timeslice-specific behavior. - """ - return self._results - - def set_network_rtt_ns(self, rtt_ns: float | None) -> None: - """Set the run-level mean network RTT (ns) to subtract from latency metrics. - - Called by the RecordsManager before summarize() when network latency - calibration is enabled (or a manual override was provided). None disables - the adjustment. - """ - self._network_rtt_ns = rtt_ns - - def _inject_network_adjusted_metrics(self) -> None: - """Inject network_adjusted_* distributions and the network_rtt summary. - - Subtracting a constant RTT from every record shifts the mean and every - percentile by that constant and leaves the standard deviation unchanged, so the - adjustment is applied to each source metric's aggregated array (clamped at 0). - Values flow through the normal _create_metric_result / to_display_unit path - because the network_adjusted_* and network_rtt metrics are registered. - """ - rtt_ns = self._network_rtt_ns - if rtt_ns is None: - return - - self._results[NetworkRttMetric.tag] = float(rtt_ns) - - clamped_tags: list[str] = [] - for adjusted_tag, source_tag in NETWORK_ADJUSTED_SOURCES.items(): - source = self._results.get(source_tag) - if not isinstance(source, MetricArray) or len(source) == 0: - continue - data = source.data - if np.any(data < rtt_ns): - clamped_tags.append(adjusted_tag) - adjusted = MetricArray() - adjusted.extend(np.maximum(data - rtt_ns, 0.0)) - self._results[adjusted_tag] = adjusted - - if clamped_tags: - self.warning( - lambda tags=clamped_tags: f"Network RTT ({rtt_ns / 1e6:.3f} ms) exceeded " - f"some measured latencies; clamped to 0 for: {', '.join(tags)}. " - "The subtracted RTT may be larger than the true network leg." - ) - - async def update_derived_metrics(self) -> None: - """Computes the values for the derived metrics, and stores them in the results dict.""" - for tag, derive_func in self.derive_funcs.items(): - try: - self._results[tag] = derive_func(self._results) - except NoMetricValue as e: - self.debug(f"No metric value for derived metric '{tag}': {e!r}") - except Exception as e: - self.warning(f"Error deriving metric '{tag}': {e!r}") - - def _should_include_in_summary(self, tag: str) -> bool: - """Check if a metric should be included in summarize() output. - - INTERNAL and EXPERIMENTAL metrics are computed (they may be dependencies - of other metrics) but filtered from output unless dev mode flags are set. - """ - metric_instance = self._instances_map[tag] - - # Filter INTERNAL metrics unless SHOW_INTERNAL_METRICS is enabled - if ( - metric_instance.has_flags(MetricFlags.INTERNAL) - and not Environment.DEV.SHOW_INTERNAL_METRICS - ): - return False - - # Filter EXPERIMENTAL metrics unless SHOW_EXPERIMENTAL_METRICS is enabled - return not ( - metric_instance.has_flags(MetricFlags.EXPERIMENTAL) - and not Environment.DEV.SHOW_EXPERIMENTAL_METRICS - ) - - async def summarize(self) -> list[MetricResult]: - """Summarize the results. - - This will compute the values for the derived metrics, and then create the MetricResult objects for each metric. - Results are returned in display units so consumers can use them directly. - - Note: INTERNAL and EXPERIMENTAL metrics are computed (as they may be dependencies) - but filtered from output unless dev mode flags are enabled. - """ - await self.update_derived_metrics() - self._inject_network_adjusted_metrics() - - # Compute metric results, filter internal/experimental, and convert to display units - results = [ - to_display_unit(self._create_metric_result(tag, values), MetricRegistry) - for tag, values in self._results.items() - if self._should_include_in_summary(tag) - ] - self.debug(lambda: f"Summarized {len(results)} metric results") - return results - - async def full_metrics(self) -> MetricResultsDict: - """Returns the full metrics dict, including the derived metrics.""" - await self.update_derived_metrics() - return self._results - - def _create_metric_result( - self, tag: MetricTagT, values: MetricDictValueTypeT - ) -> MetricResult: - """Create a MetricResult from a the current values of a metric.""" - - metric_class = self._instances_map[tag] - - if isinstance(values, MetricAggregator): - return values.to_result(tag, metric_class.header, str(metric_class.unit)) - - if isinstance(values, int | float): - return MetricResult( - tag=metric_class.tag, - header=metric_class.header, - unit=str(metric_class.unit), - avg=values, - count=1, - ) - - raise ValueError(f"Unexpected values type: {type(values)}") diff --git a/src/aiperf/post_processors/otel_metrics_results_processor.py b/src/aiperf/post_processors/otel_metrics_results_processor.py index f6adb0cd4e..4b4897772e 100644 --- a/src/aiperf/post_processors/otel_metrics_results_processor.py +++ b/src/aiperf/post_processors/otel_metrics_results_processor.py @@ -96,7 +96,7 @@ class OTelMetricsResultsProcessor(BaseMetricsProcessor): tracking server happens in a dedicated child process — see ``run_otel_streaming_fanout``. - Registered as the ``otel_metrics_streamer`` results processor in ``plugins.yaml``. + Registered as the ``otel_metrics_streamer`` stream exporter in ``plugins.yaml``. Raises ``PostProcessorDisabled`` from ``__init__`` when neither ``--otel-url`` nor ``--mlflow-tracking-uri`` is set, or when the optional ``aiperf[otel]`` extra is missing. @@ -105,9 +105,8 @@ class OTelMetricsResultsProcessor(BaseMetricsProcessor): protocol. """ - # Telemetry failures must not crash the benchmark. The records manager - # checks this attribute (see ``post_processors.protocols.BestEffortMarker`` - # and ``IS_BEST_EFFORT_ATTR``) and swallows the exception when True. + # Telemetry failures must not crash the benchmark. Stream-exporter dispatch + # returns per-handler errors instead of propagating them through the run. is_best_effort: ClassVar[bool] = True def __init__( @@ -243,7 +242,7 @@ async def _start_fanout_process(self) -> None: f"MLflow live: {self._mlflow_live_enabled})" ) - async def process_result(self, record_data: OTelResultData) -> None: + async def process_record(self, record_data: OTelResultData) -> None: """Record metric data for export via the OpenTelemetry SDK.""" if not self._streaming_ready: return @@ -259,6 +258,10 @@ async def process_result(self, record_data: OTelResultData) -> None: ) ) + async def finalize(self) -> None: + """Flush pending streaming telemetry events before results publication.""" + await self.flush(force=True) + async def flush(self, *, force: bool = False) -> None: """Force a flush of pending SDK metrics exports.""" self._queue_fanout_event("flush", {}) diff --git a/src/aiperf/post_processors/protocols.py b/src/aiperf/post_processors/protocols.py index 4ef9ff5a79..66f79c20e5 100644 --- a/src/aiperf/post_processors/protocols.py +++ b/src/aiperf/post_processors/protocols.py @@ -3,62 +3,20 @@ from __future__ import annotations -from typing import TYPE_CHECKING, ClassVar, Final, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Protocol, runtime_checkable from aiperf.common.models import ParsedResponseRecord from aiperf.common.protocols import AIPerfLifecycleProtocol -# Class-level attribute name that a results processor can set to ``True`` to -# opt out of hard-failure propagation in the records manager dispatch loop. -# Used by telemetry processors (OTel / MLflow live streaming) whose failures -# must not crash the benchmark. Centralised here so future authors can find -# the convention and type-check their use. -IS_BEST_EFFORT_ATTR: Final[str] = "is_best_effort" - - -class BestEffortMarker(Protocol): - """Marker protocol for results processors that tolerate dispatch failures. - - A processor that sets ``is_best_effort: ClassVar[bool] = True`` signals to - the records manager that ``process_result`` exceptions should be logged but - not re-raised. The records manager reads this attribute via ``getattr`` - (structural typing is advisory — runtime isinstance checks would force - every processor to inherit from a shared base). - """ - - is_best_effort: ClassVar[bool] - - if TYPE_CHECKING: - from aiperf.common.messages.inference_messages import MetricRecordsData - from aiperf.common.models import CreditPhaseStats, MetricResult from aiperf.common.models.record_models import MetricRecordMetadata from aiperf.metrics.metric_dicts import MetricRecordDict @runtime_checkable class RecordProcessorProtocol(AIPerfLifecycleProtocol, Protocol): - """Protocol for a record processor that processes the incoming records and returns the results of the post processing.""" + """Protocol for per-worker record processors that extract metric records.""" async def process_record( self, record: ParsedResponseRecord, metadata: MetricRecordMetadata ) -> MetricRecordDict: ... - - -@runtime_checkable -class ResultsProcessorProtocol(AIPerfLifecycleProtocol, Protocol): - """Protocol for a results processor that processes the results of multiple - record processors, and provides the ability to summarize the results.""" - - async def process_result( - self, record_data: MetricRecordsData | CreditPhaseStats - ) -> None: ... - - async def summarize(self) -> list[MetricResult]: ... - - -@runtime_checkable -class FlushableResultsProcessorProtocol(AIPerfLifecycleProtocol, Protocol): - """Protocol for metric results processors that support explicit flushing.""" - - async def flush(self, *, force: bool = False) -> None: ... diff --git a/src/aiperf/post_processors/record_export_jsonl_writer.py b/src/aiperf/post_processors/record_export_jsonl_writer.py index 8fc337987b..076fbd4729 100644 --- a/src/aiperf/post_processors/record_export_jsonl_writer.py +++ b/src/aiperf/post_processors/record_export_jsonl_writer.py @@ -92,14 +92,6 @@ async def process_record(self, record_data: MetricRecordsData) -> None: except Exception as e: # noqa: BLE001 - per-record; skip bad record and continue self.error(f"Failed to write record metrics: {e}") - # Registered as a ``stream_exporter`` (fed via ``process_record`` with - # record-type routing). The ``process_result`` alias is kept so the class - # also satisfies the results-processor protocol if ever dispatched that way, - # but ``record_export`` is registered ONLY as a stream_exporter -- the legacy - # ``results_processor:record_export`` (RecordExportResultsProcessor) was - # removed so the two no longer both truncate-and-write profile_export.jsonl. - process_result = process_record - async def summarize(self) -> list[MetricResult]: """No aggregation needed for JSONL export.""" return [] diff --git a/src/aiperf/post_processors/record_export_results_processor.py b/src/aiperf/post_processors/record_export_results_processor.py deleted file mode 100644 index a327dc66df..0000000000 --- a/src/aiperf/post_processors/record_export_results_processor.py +++ /dev/null @@ -1,93 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from aiperf.common.enums import ExportLevel -from aiperf.common.environment import Environment -from aiperf.common.exceptions import PostProcessorDisabled -from aiperf.common.messages.inference_messages import MetricRecordsData -from aiperf.common.mixins import BufferedJSONLWriterMixin -from aiperf.common.models.record_models import MetricRecordInfo, MetricResult -from aiperf.metrics.metric_dicts import MetricRecordDict -from aiperf.metrics.metric_registry import MetricRegistry -from aiperf.post_processors.base_metrics_processor import BaseMetricsProcessor - -if TYPE_CHECKING: - from aiperf.config.resolution.plan import BenchmarkRun - - -class RecordExportResultsProcessor( - BaseMetricsProcessor, BufferedJSONLWriterMixin[MetricRecordInfo] -): - """Exports per-record metrics to JSONL with display unit conversion and filtering.""" - - def __init__( - self, - service_id: str, - run: BenchmarkRun, - **kwargs, - ): - export_level = run.cfg.artifacts.export_level - if export_level not in (ExportLevel.RECORDS, ExportLevel.RAW): - raise PostProcessorDisabled( - f"Record export results processor is disabled for export level {export_level}" - ) - - output_file = run.cfg.artifacts.profile_export_jsonl_file - output_file.parent.mkdir(parents=True, exist_ok=True) - output_file.unlink(missing_ok=True) - - # Initialize parent classes with the output file - super().__init__( - output_file=output_file, - batch_size=Environment.RECORD.EXPORT_BATCH_SIZE, - run=run, - **kwargs, - ) - - self.show_internal = ( - Environment.DEV.MODE and Environment.DEV.SHOW_INTERNAL_METRICS - ) - self.show_experimental = ( - Environment.DEV.MODE and Environment.DEV.SHOW_EXPERIMENTAL_METRICS - ) - self.export_http_trace = run.cfg.artifacts.trace - self.info(f"Record metrics export enabled: {self.output_file}") - if self.export_http_trace: - self.info("HTTP trace export enabled (--export-http-trace)") - - async def process_result(self, record_data: MetricRecordsData) -> None: - try: - metric_dict = MetricRecordDict(record_data.metrics) - display_metrics = metric_dict.to_display_dict( - MetricRegistry, self.show_internal, self.show_experimental - ) - # Skip records with no displayable metrics UNLESS they have an error - # (error records should always be exported for debugging/analysis) - if not display_metrics and not record_data.error: - return - - # Convert trace data to export format (wall-clock timestamps) if enabled - export_trace_data = None - if self.export_http_trace and record_data.trace_data: - export_trace_data = record_data.trace_data.to_export() - - record_info = MetricRecordInfo( - metadata=record_data.metadata, - metrics=display_metrics, - trace_data=export_trace_data, - error=record_data.error, - ) - - # Write using the buffered writer mixin (handles batching and flushing) - await self.buffered_write(record_info) - - except Exception as e: - self.error(f"Failed to write record metrics: {e}") - - async def summarize(self) -> list[MetricResult]: - """Summarize the results. For this processor, we don't need to summarize anything.""" - return [] diff --git a/src/aiperf/post_processors/strategies/core.py b/src/aiperf/post_processors/strategies/core.py index 2527115940..965773fc4c 100644 --- a/src/aiperf/post_processors/strategies/core.py +++ b/src/aiperf/post_processors/strategies/core.py @@ -57,9 +57,8 @@ async def process(self, record_data: OTelResultData) -> None: Must be cheap — the caller is on the benchmark's record-processing dispatch path. Instrument access goes through the context's ``get_or_create_*`` factories (which enqueue fanout events rather than - constructing OTel SDK instruments inline). Raising is permitted; - ``OTelMetricsResultsProcessor.is_best_effort = True`` means the - records manager logs and swallows the failure. + constructing OTel SDK instruments inline). Raising is permitted; the + stream-exporter dispatcher logs handler failures and continues. """ ... diff --git a/src/aiperf/post_processors/timeslice_metric_results_processor.py b/src/aiperf/post_processors/timeslice_metric_results_processor.py deleted file mode 100644 index cd1cb60bc6..0000000000 --- a/src/aiperf/post_processors/timeslice_metric_results_processor.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - -from collections import defaultdict -from typing import TYPE_CHECKING, Any - -from aiperf.common.constants import NANOS_PER_SECOND -from aiperf.common.exceptions import NoMetricValue, PostProcessorDisabled -from aiperf.common.models import MetricResult -from aiperf.common.types import MetricTagT, TimeSliceT -from aiperf.metrics.base_metric import BaseMetric -from aiperf.metrics.display_units import to_display_unit -from aiperf.metrics.metric_dicts import MetricResultsDict -from aiperf.metrics.metric_registry import MetricRegistry -from aiperf.post_processors.metric_results_processor import MetricResultsProcessor - -if TYPE_CHECKING: - from aiperf.config.resolution.plan import BenchmarkRun - - -class TimesliceMetricResultsProcessor(MetricResultsProcessor): - """Processor for metric results in timeslice mode. - - Groups metrics by time slices based on request timestamps and slice_duration. - """ - - def __init__(self, run: BenchmarkRun, **kwargs: Any): - super().__init__(run=run, **kwargs) - - if self.run.cfg.artifacts.slice_duration is None: - raise PostProcessorDisabled( - "TimesliceMetricResultsProcessor requires slice_duration to be set" - ) - - self._slice_duration_ns: int = int( - self.run.cfg.artifacts.slice_duration * NANOS_PER_SECOND - ) - - # Set up aggregate metric object default initialization for each timeslice - self._timeslice_instances_maps: dict[ - TimeSliceT, dict[MetricTagT, BaseMetric] - ] = defaultdict( - lambda: { - tag: MetricRegistry.get_class(tag)() - for tag in MetricRegistry.all_tags() - } - ) - - # Use instance variable with defaultdict for auto-vivification - self._timeslice_results: dict[TimeSliceT, MetricResultsDict] = defaultdict( - MetricResultsDict - ) - - async def get_timeslice_index(self, request_start_ns: int): - return int(request_start_ns / self._slice_duration_ns) - - async def get_instances_map( - self, request_start_ns: int | None = None - ) -> dict[MetricTagT, BaseMetric]: - """Get the appropriate instances map based on mode.""" - """Get the results dict for the appropriate timeslice based on request timestamp.""" - if request_start_ns is None: - raise ValueError( - "TimesliceMetricResultsProcessor::get_instances_map must be passed a request_start_ns" - ) - - timeslice_index = await self.get_timeslice_index(request_start_ns) - - # Return (or create) the timeslice instances dict for this timeslice - return self._timeslice_instances_maps[timeslice_index] - - async def get_results( - self, request_start_ns: int | None = None - ) -> MetricResultsDict: - """Get the results dict for the appropriate timeslice based on request timestamp.""" - if request_start_ns is None: - raise ValueError( - "TimesliceMetricResultsProcessor::get_results must be passed a request_start_ns" - ) - - timeslice_index = await self.get_timeslice_index(request_start_ns) - - # Return (or create) the timeslice results dict for this timeslice - return self._timeslice_results[timeslice_index] - - async def update_derived_metrics(self) -> None: - for timeslice_results in self._timeslice_results.values(): - for tag, derive_func in self.derive_funcs.items(): - try: - timeslice_results[tag] = derive_func(timeslice_results) - except NoMetricValue as e: - self.debug(f"No metric value for derived metric '{tag}': {e!r}") - except Exception as e: - self.warning(f"Error deriving metric '{tag}': {e!r}") - - async def summarize(self) -> dict[TimeSliceT, list[MetricResult]]: - """Summarize timeslice results. - - Computes derived metrics, filters INTERNAL/EXPERIMENTAL metrics (unless dev - mode flags are enabled), and converts all results to display units. - """ - self.info("Summarizing timeslice metric results...") - await self.update_derived_metrics() - - # Compute and return the metric results. - timeslice_metric_results = {} - - # Start timeslice indices at zero - for counter, timeslice_index in enumerate[TimeSliceT]( - sorted(self._timeslice_results.keys()) - ): - # Filter internal/experimental metrics and convert to display units - metric_results = [ - to_display_unit(self._create_metric_result(tag, values), MetricRegistry) - for tag, values in self._timeslice_results[timeslice_index].items() - if self._should_include_in_summary(tag) - ] - timeslice_metric_results[counter] = metric_results - - self.info( - f"Summarized {len(timeslice_metric_results)} timeslice metric results" - ) - return timeslice_metric_results diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 0b0805d56a..958b6ab8d5 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -6,7 +6,7 @@ import time from collections import defaultdict from dataclasses import dataclass, field -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from aiperf.common.accumulator_protocols import ( AccumulatorProtocol, @@ -22,7 +22,6 @@ MessageType, ) from aiperf.common.environment import Environment -from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.hooks import background_task, on_command, on_message, on_pull_message from aiperf.common.messages import ( AllRecordsReceivedMessage, @@ -47,18 +46,14 @@ from aiperf.common.mixins import PullClientMixin from aiperf.common.models import ( BranchStats, - CreditPhaseStats, ErrorDetails, ErrorDetailsCount, MetricResult, - NetworkLatencySample, PhaseRecordsStats, ProcessRecordsResult, ProcessServerMetricsResult, ProcessTelemetryResult, ProfileResults, - ServerMetricsRecord, - TelemetryRecord, TimesliceResult, WorkerProcessingStats, ) @@ -72,51 +67,34 @@ CreditPhaseStartMessage, CreditsCompleteMessage, ) -from aiperf.gpu_telemetry.protocols import ( - GPUTelemetryAccumulatorProtocol, - GPUTelemetryProcessorProtocol, -) +from aiperf.gpu_telemetry.protocols import GPUTelemetryAccumulatorProtocol from aiperf.metrics.accumulator_models import AccumulatorMetricsSummary from aiperf.metrics.cache_reporting_hint import ( CACHE_REPORTING_HINT, usage_without_cache_in_record, ) from aiperf.network_latency.accumulator import NetworkLatencyAccumulator -from aiperf.network_latency.protocols import NetworkLatencyProcessorProtocol from aiperf.plugin import plugins from aiperf.plugin.enums import ( AccumulatorType, PluginType, - ResultsProcessorType, StreamExporterType, UIType, ) -from aiperf.post_processors.metric_results_processor import MetricResultsProcessor -from aiperf.post_processors.protocols import ( - IS_BEST_EFFORT_ATTR, - FlushableResultsProcessorProtocol, - ResultsProcessorProtocol, -) from aiperf.records import records_manager_processing from aiperf.records.dataset_gate import await_dataset_configured from aiperf.records.error_tracker import ErrorTracker from aiperf.records.records_manager_processing import ( - accumulators_for_record_type, generate_realtime_metrics, load_accumulators, load_stream_exporters, - stream_exporters_for_record_type, ) from aiperf.records.records_tracker import RecordsTracker -from aiperf.server_metrics.protocols import ( - ServerMetricsAccumulatorProtocol, - ServerMetricsProcessorProtocol, -) +from aiperf.server_metrics.protocols import ServerMetricsAccumulatorProtocol if TYPE_CHECKING: from aiperf.config.config import BenchmarkConfig from aiperf.config.resolution.plan import BenchmarkRun - from aiperf.plugin.types import PluginEntry _LATENCY_LINE_LABELS: tuple[tuple[str, str], ...] = ( @@ -442,17 +420,12 @@ def __init__( self._server_metrics_state = ErrorTrackingState() self._metric_state = ErrorTrackingState() - self._metric_results_processors: list[ResultsProcessorProtocol] = [] # fmt: skip - self._timing_results_processors: list[ResultsProcessorProtocol] = [] # fmt: skip - self._gpu_telemetry_processors: list[GPUTelemetryProcessorProtocol] = [] # fmt: skip - self._server_metrics_processors: list[ServerMetricsProcessorProtocol] = [] # fmt: skip - self._gpu_telemetry_accumulator: GPUTelemetryAccumulatorProtocol | None = None # fmt: skip - self._server_metrics_accumulator: ServerMetricsAccumulatorProtocol | None = None # fmt: skip - self._network_latency_processors: list[NetworkLatencyProcessorProtocol] = [] # fmt: skip + self._gpu_telemetry_accumulator: GPUTelemetryAccumulatorProtocol | None = None + self._server_metrics_accumulator: ServerMetricsAccumulatorProtocol | None = None # In-process accumulator for RTT probe samples. Computes the run-level - # mean RTT delivered to each MetricResultsProcessor via set_network_rtt_ns - # before summarize(). None unless network latency probing is active. + # mean RTT delivered to MetricsAccumulator before summarize(). None unless + # network latency probing is active. self._network_latency_accumulator: NetworkLatencyAccumulator | None = ( NetworkLatencyAccumulator(benchmark_id=self.run.benchmark_id) if self.run.cfg.network_latency.should_probe @@ -460,119 +433,92 @@ def __init__( ) self._network_latency_state = ErrorTrackingState() - for entry in plugins.iter_entries(PluginType.RESULTS_PROCESSOR): - try: - ProcessorClass = plugins.get_class( - PluginType.RESULTS_PROCESSOR, entry.name - ) - results_processor = ProcessorClass( - service_id=self.service_id, - run=self.run, - pub_client=self.pub_client, - ) - self.attach_child_lifecycle(results_processor) - - self._classify_results_processor(results_processor, entry) - - self.debug( - f"Created results processor: {entry.name}: {results_processor.__class__.__name__}" - ) - except PostProcessorDisabled as e: - if entry.name == ResultsProcessorType.OTEL_METRICS_STREAMER: - self.info( - f"OTel metrics streamer is disabled and will not be used: {e}" - ) - else: - self.debug( - f"Results processor {entry.name} is disabled and will not be used" - ) - except Exception as e: - self.error(f"Failed to create results processor {entry.name}: {e}") - - # --- agentx accumulator pipeline (the byte-exact summary engine) ------- - # The MetricsAccumulator (accumulator:metric_results) is the SOLE summary - # producer for the records pipeline — the final ProfileResults come only - # from summarizing it. The legacy results_processor:metric_results - # (MetricResultsProcessor) is never summarized (no summarize() is ever - # called on the _metric_results_processors list), so it is a dead - # per-record dispatch once the accumulator is present and there is NO - # fallback summary path if the accumulator is missing. Telemetry / - # server-metrics keep flowing through main's gpu_telemetry_processor / - # server_metrics_processor side-channels for #803 efficiency metrics — - # the accumulator drive here is metric_records-only. self._accumulators: dict[AccumulatorType, AccumulatorProtocol] = ( load_accumulators(self) ) self._stream_exporters: dict[StreamExporterType, StreamExporterProtocol] = ( load_stream_exporters(self) ) + self._routing_table = self._build_routing_table() + self._log_routing_table() - # Pre-resolve the metric_records dispatch lists once (per-record hot path). - self._metric_record_accumulators = accumulators_for_record_type( - self._accumulators, "metric_records" - ) - self._metric_record_stream_exporters = stream_exporters_for_record_type( - self._stream_exporters, "metric_records" - ) - # GPU-telemetry / server-metrics JSONL writers are stream_exporters fed by - # record-type routing (mirrors agentx). The accumulators stay in the - # gpu_telemetry_processor / server_metrics_processor side-channels; only - # the per-record JSONL writers live here. - self._gpu_telemetry_stream_exporters = stream_exporters_for_record_type( - self._stream_exporters, "gpu_telemetry" + self._metric_record_accumulators = [ + accumulator + for accumulator in self._accumulators.values() + if accumulator in self._routing_table.get("metric_records", []) + ] + self._gpu_telemetry_accumulator = self._accumulators.get( + AccumulatorType.GPU_TELEMETRY ) - self._server_metrics_stream_exporters = stream_exporters_for_record_type( - self._stream_exporters, "server_metrics" + self._server_metrics_accumulator = self._accumulators.get( + AccumulatorType.SERVER_METRICS ) - # The accumulator is the sole summary producer, and the legacy - # MetricResultsProcessor is never summarized — so gating it off simply - # removes a dead per-record dispatch (it is not a fallback and does not - # otherwise contribute to the results). - if AccumulatorType.METRIC_RESULTS in self._accumulators: - # Exact-type check, deliberately NOT isinstance: subclasses like - # TimesliceMetricResultsProcessor (results_processor:timeslice) - # produce non-summary outputs and must stay active. - self._metric_results_processors = [ - p - for p in self._metric_results_processors - if type(p) is not MetricResultsProcessor - ] - self.debug( - "MetricsAccumulator active; legacy MetricResultsProcessor gated off" + def _build_routing_table(self) -> dict[str, list[Any]]: + """Build record_type string -> handler mapping from plugin metadata.""" + table: dict[str, list[Any]] = {} + for entry in plugins.iter_entries(PluginType.ACCUMULATOR): + handler = self._accumulators.get(AccumulatorType(entry.name)) + if handler is None: + continue + record_types = ( + entry.metadata.get("record_types", []) if entry.metadata else [] ) + for record_type in record_types: + table.setdefault(record_type, []).append(handler) - def _classify_results_processor( - self, - results_processor: ResultsProcessorProtocol, - entry: PluginEntry, - ) -> None: - """Route a constructed results processor into its subsystem bucket. - - Sorts by protocol into GPU telemetry, server metrics, network latency, - or the generic metric processors list, and captures the GPU/server - accumulator singletons and any timing-capable processor. - """ - if isinstance(results_processor, GPUTelemetryProcessorProtocol): - self._gpu_telemetry_processors.append(results_processor) - if entry.name == ResultsProcessorType.GPU_TELEMETRY_ACCUMULATOR: - self._gpu_telemetry_accumulator = results_processor - - elif isinstance(results_processor, ServerMetricsProcessorProtocol): - self._server_metrics_processors.append(results_processor) - if entry.name == ResultsProcessorType.SERVER_METRICS_ACCUMULATOR: - self._server_metrics_accumulator = results_processor + for entry in plugins.iter_entries(PluginType.STREAM_EXPORTER): + handler = self._stream_exporters.get(StreamExporterType(entry.name)) + if handler is None: + continue + record_types = ( + entry.metadata.get("record_types", []) if entry.metadata else [] + ) + for record_type in record_types: + table.setdefault(record_type, []).append(handler) + return table + + async def _dispatch_record(self, record: Any) -> list[BaseException]: + """Dispatch one typed record to all handlers registered for its record_type.""" + record_type = getattr(record, "record_type", None) + if record_type is None: + error = TypeError(f"Record {type(record).__name__} has no record_type") + self.error(str(error)) + return [error] + + handlers = self._routing_table.get(record_type, []) + if not handlers: + self.debug(lambda: f"No handlers registered for record type: {record_type}") + return [] - elif isinstance(results_processor, NetworkLatencyProcessorProtocol): - self._network_latency_processors.append(results_processor) + results = await asyncio.gather( + *[handler.process_record(record) for handler in handlers], + return_exceptions=True, + ) + errors: list[BaseException] = [] + for handler, result in zip(handlers, results, strict=True): + if isinstance(result, asyncio.CancelledError): + raise result + if isinstance(result, BaseException): + self.error( + f"Handler {handler.__class__.__name__} failed for " + f"{record_type}: {result!r}" + ) + errors.append(result) + return errors - else: - self._metric_results_processors.append(results_processor) - if ( - entry.name == ResultsProcessorType.OTEL_METRICS_STREAMER - and self.run.cfg.otel.stream_timing_enabled - ): - self._timing_results_processors.append(results_processor) + def _log_routing_table(self) -> None: + """Log the metadata-derived record routing table.""" + self.debug( + lambda: ( + f"Routing table: {len(self._accumulators)} accumulators, " + f"{len(self._stream_exporters)} stream exporters, " + f"{len(self._routing_table)} record types" + ) + ) + for record_type, handlers in self._routing_table.items(): + handler_names = [handler.__class__.__name__ for handler in handlers] + self.debug(lambda rt=record_type, hn=handler_names: f" {rt} -> {hn}") @on_pull_message(MessageType.METRIC_RECORDS) async def _on_metric_records(self, message: MetricRecordsMessage) -> None: @@ -586,140 +532,61 @@ async def _on_metric_records(self, message: MetricRecordsMessage) -> None: self._maybe_hint_missing_cache_reporting(record_data) - # Drive the accumulator engine (primary summary path) AND any legacy - # results_processors that survived the accumulator gate (e.g. otel - # streaming, which is best-effort and does not touch summary numbers). - await self._send_record_to_accumulators(record_data) - # A non-best-effort results processor that raises must NOT skip the - # tracker update + completion-barrier check, or the phase never - # converges and the (timeout-less) barrier hangs. Run those in a - # finally, then let the original exception re-propagate (already logged - # inside _send_results_to_results_processors). - try: - await self._send_results_to_results_processors(record_data) - finally: - self._records_tracker.update_from_record_data(record_data) - if record_data.error: - self._error_tracker.increment_error_count_for_phase( - record_data.metadata.benchmark_phase, record_data.error - ) - - phase = record_data.metadata.benchmark_phase - if ( - phase in self._complete_credit_phases - and self._records_tracker.check_and_set_all_records_received_for_phase( - phase - ) - ): - await self._handle_all_records_received(phase) - - async def _send_record_to_accumulators( - self, record_data: MetricRecordsData - ) -> None: - """Dispatch a metric record to all metric_records accumulators + stream exporters. + await self._dispatch_record(record_data) + self._records_tracker.update_from_record_data(record_data) + if record_data.error: + self._error_tracker.increment_error_count_for_phase( + record_data.metadata.benchmark_phase, record_data.error + ) - Per-handler exceptions are caught so one bad accumulator does not abort - the others. GPU telemetry / server metrics records are routed via their - own ``@on_pull_message`` handlers (and main's side-channel processors) - and do not flow through here. - """ - targets: list[object] = [ - *self._metric_record_accumulators, - *self._metric_record_stream_exporters, - ] - if not targets: - return - results = await asyncio.gather( - *[t.process_record(record_data) for t in targets], - return_exceptions=True, - ) - for target, result in zip(targets, results, strict=True): - if isinstance(result, BaseException): - self.error( - f"Accumulator {target.__class__.__name__} failed for " - f"metric_records: {result!r}" - ) + phase = record_data.metadata.benchmark_phase + if ( + phase in self._complete_credit_phases + and self._records_tracker.check_and_set_all_records_received_for_phase( + phase + ) + ): + await self._handle_all_records_received(phase) @on_pull_message(MessageType.TELEMETRY_RECORDS) async def _on_telemetry_records(self, message: TelemetryRecordsMessage) -> None: - """Handle telemetry records message from Telemetry Manager. - The RecordsManager acts as the central hub for all record processing, - whether inference metrics or GPU telemetry. - - Args: - message: Batch of telemetry records from a DCGM collector - """ + """Handle telemetry records message from Telemetry Manager.""" if message.valid: - try: - await self._send_telemetry_to_results_processors(message.records) - except Exception as e: - error_details = ErrorDetails( - message=f"Telemetry processor error: {str(e)}" - ) - self._telemetry_state.error_counts[error_details] += 1 - self.debug(f"Failed to process telemetry batch: {e}") - else: - if message.error: - self._telemetry_state.error_counts[message.error] += 1 + for record in message.records: + for error in await self._dispatch_record(record): + self._telemetry_state.error_counts[ + ErrorDetails.from_exception(error) + ] += 1 + elif message.error: + self._telemetry_state.error_counts[message.error] += 1 @on_pull_message(MessageType.SERVER_METRICS_RECORD) async def _on_server_metrics_records( self, message: ServerMetricsRecordMessage ) -> None: - """Handle server metrics record message from Server Metrics Manager. - - Forwards full record to results processors. - - Args: - message: Server metrics record from a Prometheus collector - """ + """Handle server metrics record message from Server Metrics Manager.""" if message.valid: - # Forward full records to results processors - await self._send_server_metrics_to_results_processors(message.record) - else: - if message.error: - self._server_metrics_state.error_counts[message.error] += 1 + for error in await self._dispatch_record(message.record): + self._server_metrics_state.error_counts[ + ErrorDetails.from_exception(error) + ] += 1 + elif message.error: + self._server_metrics_state.error_counts[message.error] += 1 @on_pull_message(MessageType.NETWORK_LATENCY_RECORD) async def _on_network_latency_records( self, message: NetworkLatencyRecordMessage ) -> None: - """Handle a network latency RTT probe sample from the NetworkLatencyManager. - - Accumulates the sample for the run-level mean RTT (delivered to the - metric processors before summarize) and forwards it to the JSONL writer. - A transport-level delivery error is tracked separately. - - Args: - message: Network latency probe sample from a probe collector - """ + """Handle a network latency RTT probe sample from the NetworkLatencyManager.""" if message.valid: if self._network_latency_accumulator is not None: self._network_latency_accumulator.add_sample(message.sample) - await self._send_network_latency_to_results_processors(message.sample) - else: - if message.error: - self._network_latency_state.error_counts[message.error] += 1 - - async def _send_network_latency_to_results_processors( - self, sample: NetworkLatencySample - ) -> None: - """Forward a probe sample to the network latency results processors.""" - if not self._network_latency_processors: - return - errors = await asyncio.gather( - *[ - processor.process_network_latency_sample(sample) - for processor in self._network_latency_processors - ], - return_exceptions=True, - ) - for error in errors: - if isinstance(error, BaseException): - self.exception(f"Failed to process network latency sample: {error!r}") + for error in await self._dispatch_record(message.sample): self._network_latency_state.error_counts[ ErrorDetails.from_exception(error) ] += 1 + elif message.error: + self._network_latency_state.error_counts[message.error] += 1 async def _handle_all_records_received(self, phase: CreditPhase) -> None: """Handle the case where all records have been received.""" @@ -793,178 +660,13 @@ async def _finalize_and_process_results( await self._process_results(phase=phase, cancelled=cancelled) self.info("_finalize_and_process_results completed") - async def _send_results_to_results_processors( - self, record_data: MetricRecordsData - ) -> None: - """Send the results to each of the metric results processors. - - Telemetry-only processors (FlushableResultsProcessorProtocol, e.g. OTel - streaming) are best-effort: their exceptions are logged but do not fail - the run. All other processors propagate exceptions so data-pipeline - failures surface immediately. - """ - if not self._metric_results_processors: - return - - for results_processor in self._metric_results_processors: - try: - await results_processor.process_result(record_data) - # telemetry processor failure must not crash the run - except Exception as exc: - self.exception( - "Failed to process metric record in " - f"{results_processor.__class__.__name__}: {exc!r}" - ) - # Processors with is_best_effort=True (streaming telemetry like - # OTel) swallow exceptions; all others re-raise to surface bugs. - # See ``post_processors.protocols.BestEffortMarker``. - if not getattr(results_processor, IS_BEST_EFFORT_ATTR, False): - raise - - async def _flush_metric_results_processors(self, force: bool = False) -> None: - """Flush any results processors that provide explicit flush support. - - Mirrors the best-effort contract from ``_send_results_to_results_processors``: - flush failures on processors marked ``is_best_effort=True`` (e.g. OTel - streaming) are logged and swallowed; non-best-effort processors re-raise - so data-pipeline bugs surface. Today every ``FlushableResultsProcessorProtocol`` - implementer is best-effort telemetry, but the explicit check keeps the - contract consistent with the per-record path if a future flushable - processor (e.g. a Parquet writer) is added. - """ - flushable_processors = [ - results_processor - for results_processor in self._metric_results_processors - if isinstance(results_processor, FlushableResultsProcessorProtocol) - ] - if not flushable_processors: - return - - self.debug( - lambda: f"Flushing {len(flushable_processors)} metric results processors" - ) - results = await asyncio.gather( - *[processor.flush(force=force) for processor in flushable_processors], - return_exceptions=True, - ) - for processor, result in zip(flushable_processors, results, strict=True): - if not isinstance(result, BaseException): - continue - self.exception( - f"Failed to flush metric results processor " - f"{processor.__class__.__name__}: {result!r}" - ) - if not getattr(processor, IS_BEST_EFFORT_ATTR, False): - raise result - - async def _send_timing_to_results_processors( - self, phase_stats: CreditPhaseStats - ) -> None: - """Send timing snapshots to timing-capable results processors. - - Mirrors the best-effort contract from ``_send_results_to_results_processors``: - failures on processors marked ``is_best_effort=True`` are logged and - swallowed; non-best-effort failures re-raise. All timing processors - today are OTel streaming telemetry (best-effort), but the explicit - check keeps the behaviour predictable if a non-telemetry timing - processor is added later. - """ - if not self._timing_results_processors: - return - - results = await asyncio.gather( - *[ - results_processor.process_result(phase_stats) - for results_processor in self._timing_results_processors - ], - return_exceptions=True, - ) - for results_processor, result in zip( - self._timing_results_processors, results, strict=True - ): - if not isinstance(result, BaseException): - continue - self.exception( - "Failed to process timing snapshot in " - f"{results_processor.__class__.__name__}: {result!r}" - ) - if not getattr(results_processor, IS_BEST_EFFORT_ATTR, False): - raise result - - async def _send_telemetry_to_results_processors( - self, telemetry_records: list[TelemetryRecord] - ) -> None: - """Send individual telemetry records to telemetry results processors only. - - Args: - telemetry_records: Batch of records from single collection cycle - """ - errors = await asyncio.gather( - *[ - processor.process_telemetry_record(record) - for processor in self._gpu_telemetry_processors - for record in telemetry_records # Process each record individually - ], - return_exceptions=True, - ) - for error in errors: - if isinstance(error, BaseException): - self.exception(f"Failed to process telemetry record: {error!r}") - self._telemetry_state.error_counts[ - ErrorDetails.from_exception(error) - ] += 1 - for exporter in self._gpu_telemetry_stream_exporters: - for record in telemetry_records: - try: - await exporter.process_record(record) - except Exception as exc: - self.error( - f"Stream exporter {exporter.__class__.__name__} failed for " - f"gpu_telemetry record: {exc!r}" - ) - - async def _send_server_metrics_to_results_processors( - self, record: ServerMetricsRecord - ) -> None: - """Send individual server metrics records to server metrics results processors only. - - Args: - record: ServerMetricsRecord from single collection cycle - """ - errors = await asyncio.gather( - *[ - processor.process_server_metrics_record(record) - for processor in self._server_metrics_processors - ], - return_exceptions=True, - ) - for error in errors: - if isinstance(error, BaseException): - self.exception(f"Failed to process server metrics record: {error!r}") - self._server_metrics_state.error_counts[ - ErrorDetails.from_exception(error) - ] += 1 - for exporter in self._server_metrics_stream_exporters: - try: - await exporter.process_record(record) - except Exception as exc: - self.error( - f"Stream exporter {exporter.__class__.__name__} failed for " - f"server_metrics record: {exc!r}" - ) - @on_message(MessageType.DATASET_CONFIGURED_NOTIFICATION) async def _on_dataset_configured( self, message: DatasetConfiguredNotification ) -> None: - for processor in self._metric_results_processors: - if hasattr(processor, "on_dataset_configured"): - processor.on_dataset_configured(message.metadata) - # Accumulators that consume loader-stamped per-turn metadata - # (e.g. TheoreticalPrefixCacheAccumulator) need the dataset config too. - for accumulator in self._accumulators.values(): - if hasattr(accumulator, "on_dataset_configured"): - accumulator.on_dataset_configured(message.metadata) + for handler in (*self._accumulators.values(), *self._stream_exporters.values()): + if hasattr(handler, "on_dataset_configured"): + handler.on_dataset_configured(message.metadata) self._dataset_configured_event.set() @on_message(MessageType.CREDIT_PHASE_START) @@ -973,7 +675,7 @@ async def _on_credit_phase_start( ) -> None: """Handle a credit phase start message in order to track the total number of expected requests.""" self._records_tracker.update_phase_info(phase_start_msg.stats) - await self._send_timing_to_results_processors(phase_start_msg.stats) + await self._dispatch_record(phase_start_msg.stats) self.info(f"Credit phase start: {phase_start_msg.config.phase}") @on_message(MessageType.CREDIT_PHASE_PROGRESS) @@ -982,7 +684,7 @@ async def _on_credit_phase_progress( ) -> None: """Handle a credit phase progress message to track and stream live timing snapshots.""" self._records_tracker.update_phase_info(message.stats) - await self._send_timing_to_results_processors(message.stats) + await self._dispatch_record(message.stats) @on_message(MessageType.CREDIT_PHASE_SENDING_COMPLETE) async def _on_credit_phase_sending_complete( @@ -994,7 +696,7 @@ async def _on_credit_phase_sending_complete( f"Sent {message.stats.final_requests_sent:,} requests. Waiting for all to complete..." ) self._records_tracker.update_phase_info(message.stats) - await self._send_timing_to_results_processors(message.stats) + await self._dispatch_record(message.stats) @on_message(MessageType.CREDIT_PHASE_COMPLETE) async def _on_credit_phase_complete( @@ -1002,7 +704,7 @@ async def _on_credit_phase_complete( ) -> None: """Handle a credit phase complete message in order to track the end time, and check if all records have been received.""" self._records_tracker.update_phase_info(message.stats) - await self._send_timing_to_results_processors(message.stats) + await self._dispatch_record(message.stats) self._complete_credit_phases.add(message.stats.phase) # Capture per-phase BranchStats for any phase that publishes them. if message.branch_stats is not None: @@ -1368,18 +1070,16 @@ async def _summarize_one_accumulator( Returns the result (or exception object) so a single bad accumulator cannot abort the rest. Accumulators that support phase/window-scoped - export (MetricsAccumulator, TheoreticalPrefixCacheAccumulator) get - ``export_results(ctx)`` so warmup records are excluded from profiling - summaries; otherwise prefers ``summarize()`` and falls back to - ``export_results(ctx)``. + export (MetricsAccumulator) get ``export_results(ctx)`` so warmup records + are excluded from profiling summaries; otherwise prefers ``summarize()`` + and falls back to ``export_results(ctx)``. """ name = accumulator.__class__.__name__ self.debug(f"Starting summarize for accumulator {acc_type}: {name}") try: - if accumulator.__class__.__name__ in { - "MetricsAccumulator", - "TheoreticalPrefixCacheAccumulator", - } and hasattr(accumulator, "export_results"): + if accumulator.__class__.__name__ == "MetricsAccumulator" and hasattr( + accumulator, "export_results" + ): res = await asyncio.wait_for( accumulator.export_results(ctx), timeout=Environment.RECORD.PROCESS_RECORDS_TIMEOUT, @@ -1534,10 +1234,10 @@ async def _publish_all_results( except Exception as e: # noqa: BLE001 - publish failure must not abort the per-record result path self.error(f"Failed to publish ProcessAllResultsMessage: {e!r}") - def _deliver_network_rtt_to_processors(self) -> None: - """Set the run-level mean network RTT (ns) on each metric results processor. + def _deliver_network_rtt_to_accumulators(self) -> None: + """Set the run-level mean network RTT (ns) on each metric-record accumulator. - Two cases, resolved here just before MetricResultsProcessor.summarize(): + Two cases, resolved here just before MetricsAccumulator.summarize(): 1. Manual mean (``--network-latency-mean``): if ``network_latency.mean_ms`` is set, the NetworkLatencyManager service is never spawned; convert the @@ -1586,13 +1286,9 @@ def _deliver_network_rtt_to_processors(self) -> None: "from latency metrics (network_adjusted_* metrics)." ) - # Deliver to the legacy results processors (if any survive) AND the - # primary MetricsAccumulator engine, which injects network_adjusted_* - # in its own summarize() from the columnar latency arrays. - for target in ( - *self._metric_results_processors, - *self._metric_record_accumulators, - ): + # Deliver to the primary MetricsAccumulator engine, which injects + # network_adjusted_* in its own summarize() from the columnar latency arrays. + for target in self._metric_record_accumulators: set_rtt = getattr(target, "set_network_rtt_ns", None) if callable(set_rtt): set_rtt(rtt_ns) @@ -1600,23 +1296,13 @@ def _deliver_network_rtt_to_processors(self) -> None: async def _process_results( self, phase: CreditPhase, cancelled: bool ) -> ProcessRecordsResult: - """Process the results. - - The MetricsAccumulator engine is the primary (byte-exact) summary - source. Any surviving best-effort results_processors (e.g. otel - streaming) are flushed so their side-channel export completes, but - they do not contribute to the summary records. - """ + """Process the accumulated records into final benchmark results.""" self.debug(lambda: f"Processing records (cancelled: {cancelled})") self.info("Processing records results...") - # Flush legacy best-effort processors (otel streaming) so their export - # completes; they do not feed the summary numbers. - await self._flush_metric_results_processors(force=True) - - # Deliver the run-level mean network RTT to each surviving metric results - # processor BEFORE summarize() so network_adjusted_* metrics can be injected. - self._deliver_network_rtt_to_processors() + # Deliver the run-level mean network RTT before summarize() so + # network_adjusted_* metrics can be injected. + self._deliver_network_rtt_to_accumulators() ( records_results, diff --git a/src/aiperf/records/records_manager_processing.py b/src/aiperf/records/records_manager_processing.py index 0150d04932..4474ee77f0 100644 --- a/src/aiperf/records/records_manager_processing.py +++ b/src/aiperf/records/records_manager_processing.py @@ -133,38 +133,6 @@ def load_stream_exporters( return exporters -def accumulators_for_record_type( - accumulators: dict[AccumulatorType, AccumulatorProtocol], - record_type: str, -) -> list[AccumulatorProtocol]: - """Return accumulators whose plugin metadata declares ``record_type``.""" - matched: list[AccumulatorProtocol] = [] - for entry in plugins.iter_entries(PluginType.ACCUMULATOR): - record_types = entry.metadata.get("record_types", []) if entry.metadata else [] - if record_type not in record_types: - continue - acc_type = AccumulatorType(entry.name) - if acc_type in accumulators: - matched.append(accumulators[acc_type]) - return matched - - -def stream_exporters_for_record_type( - exporters: dict[StreamExporterType, StreamExporterProtocol], - record_type: str, -) -> list[StreamExporterProtocol]: - """Return stream exporters whose plugin metadata declares ``record_type``.""" - matched: list[StreamExporterProtocol] = [] - for entry in plugins.iter_entries(PluginType.STREAM_EXPORTER): - record_types = entry.metadata.get("record_types", []) if entry.metadata else [] - if record_type not in record_types: - continue - exp_type = StreamExporterType(entry.name) - if exp_type in exporters: - matched.append(exporters[exp_type]) - return matched - - async def generate_realtime_metrics( accumulators: list[AccumulatorProtocol], timeout: float = 30.0, diff --git a/src/aiperf/server_metrics/protocols.py b/src/aiperf/server_metrics/protocols.py index 6c4c7cf22e..c676883b60 100644 --- a/src/aiperf/server_metrics/protocols.py +++ b/src/aiperf/server_metrics/protocols.py @@ -11,55 +11,27 @@ MetricResult, ServerMetricsRecord, ServerMetricsResults, - TimeRangeFilter, ) @runtime_checkable -class ServerMetricsProcessorProtocol(Protocol): - """Protocol for server metrics results processors that handle ServerMetricsRecord objects. +class ServerMetricsAccumulatorProtocol(Protocol): + """Protocol for server metrics accumulators and realtime exporters.""" - This protocol is separate from ResultsProcessorProtocol because server metrics data - has fundamentally different structure (hierarchical Prometheus snapshots) compared - to inference metrics (flat key-value pairs). - """ - - async def process_server_metrics_record(self, record: ServerMetricsRecord) -> None: - """Process individual server metrics record with complete Prometheus snapshot. - - Args: - record: ServerMetricsRecord containing Prometheus metrics snapshot and metadata - """ + async def process_record(self, record: ServerMetricsRecord) -> None: + """Process one Prometheus server-metrics snapshot.""" ... async def summarize(self) -> list[MetricResult]: ... - -@runtime_checkable -class ServerMetricsAccumulatorProtocol(ServerMetricsProcessorProtocol, Protocol): - """Protocol for server metrics accumulators that accumulate server metrics data and export aggregated results. - - Extends ServerMetricsProcessorProtocol to provide result export functionality with time filtering - and error summary support. Implementations should accumulate Prometheus snapshot data and compute - aggregated statistics (mean, p50, p90, p95, p99) for configured metrics across collection windows. - """ - async def export_results( self, start_ns: int, end_ns: int, - time_filter: TimeRangeFilter | None = None, error_summary: list[ErrorDetailsCount] | None = None, + *, + warmup_start_ns: int | None = None, + warmup_end_ns: int | None = None, ) -> ServerMetricsResults | None: - """Export accumulated server metrics as results. - - Args: - start_ns: Start time of collection in nanoseconds - end_ns: End time of collection in nanoseconds - time_filter: Optional time filter for aggregation (excludes warmup/buffer) - error_summary: Optional list of error counts - - Returns: - ServerMetricsResults if data was collected, None otherwise - """ + """Export accumulated server metrics for a profiling window.""" ... diff --git a/tests/unit/accuracy/test_accuracy_record_processor.py b/tests/unit/accuracy/test_accuracy_record_processor.py index 724cd6bd47..198139a289 100644 --- a/tests/unit/accuracy/test_accuracy_record_processor.py +++ b/tests/unit/accuracy/test_accuracy_record_processor.py @@ -45,7 +45,7 @@ def _make_processor(monkeypatch) -> AccuracyRecordProcessor: return AccuracyRecordProcessor(run=_make_run(), service_id="test") -def _make_results_processor() -> AccuracyResultsProcessor: +def _make_accuracy_accumulator() -> AccuracyResultsProcessor: return AccuracyResultsProcessor(run=_make_run()) @@ -72,7 +72,7 @@ def _make_record_data( ) -> MetricRecordsData: return MetricRecordsData( metadata=create_metric_metadata(session_num=session_num), - metrics={"accuracy.correct": correct, "accuracy.unparsed": unparsed}, + metrics={"accuracy_correct": correct, "accuracy_unparsed": unparsed}, ) @@ -127,8 +127,8 @@ async def test_process_record_wraps_when_session_num_exceeds_dataset( metadata = create_metric_metadata(session_num=1) result = await processor.process_record(sample_parsed_record, metadata) - assert result["accuracy.correct"] == 1.0 - assert result["accuracy.unparsed"] == 0.0 + assert result["accuracy_correct"] == 1.0 + assert result["accuracy_unparsed"] == 0.0 processor.grader.grade.assert_awaited_once_with("Hello world", "A") async def test_process_record_wraps_to_correct_problem( @@ -152,8 +152,8 @@ async def test_process_record_wraps_to_correct_problem( metadata = create_metric_metadata(session_num=4) result = await processor.process_record(sample_parsed_record, metadata) - assert result["accuracy.correct"] == 0.0 - assert result["accuracy.unparsed"] == 1.0 + assert result["accuracy_correct"] == 0.0 + assert result["accuracy_unparsed"] == 1.0 processor.grader.grade.assert_awaited_once_with("Hello world", "B") async def test_process_record_last_valid_session_num_succeeds( @@ -174,8 +174,8 @@ async def test_process_record_last_valid_session_num_succeeds( metadata = create_metric_metadata(session_num=1) result = await processor.process_record(sample_parsed_record, metadata) - assert result["accuracy.correct"] == 1.0 - assert result["accuracy.unparsed"] == 0.0 + assert result["accuracy_correct"] == 1.0 + assert result["accuracy_unparsed"] == 0.0 async def test_process_record_raises_if_not_configured( self, monkeypatch, sample_parsed_record @@ -246,7 +246,7 @@ def test_non_verbose_debug_enabled_logs_at_debug(self, monkeypatch) -> None: class TestAccuracyResultsProcessorOnDatasetConfigured: def test_populates_tasks_from_metadata(self) -> None: - processor = _make_results_processor() + processor = _make_accuracy_accumulator() metadata = _make_dataset_metadata(["A", "B"], ["algebra", "history"]) processor.on_dataset_configured(metadata) @@ -254,7 +254,7 @@ def test_populates_tasks_from_metadata(self) -> None: assert processor._tasks == ["algebra", "history"] def test_skips_conversations_without_accuracy_task(self) -> None: - processor = _make_results_processor() + processor = _make_accuracy_accumulator() conversations = [ ConversationMetadata(conversation_id="plain"), ConversationMetadata( @@ -275,90 +275,90 @@ def test_skips_conversations_without_accuracy_task(self) -> None: @pytest.mark.asyncio class TestAccuracyResultsProcessorSessionBounds: - async def test_process_result_wraps_when_session_num_exceeds_dataset(self) -> None: + async def test_process_record_wraps_when_session_num_exceeds_dataset(self) -> None: """session_num >= dataset size wraps via modulo so the correct task is recorded.""" - processor = _make_results_processor() + processor = _make_accuracy_accumulator() processor._tasks = ["algebra"] # session_num=1 wraps to index 0 (the only task, "algebra") - await processor.process_result(_make_record_data(session_num=1)) + await processor.process_record(_make_record_data(session_num=1)) assert processor._task_total["algebra"] == 1 assert processor._overall_total == 1 - async def test_process_result_wraps_to_correct_task(self) -> None: + async def test_process_record_wraps_to_correct_task(self) -> None: """With N problems, session_num=N+1 accumulates under the task at index 1.""" - processor = _make_results_processor() + processor = _make_accuracy_accumulator() processor._tasks = ["algebra", "history", "biology"] # session_num=4 % 3 = index 1 → task="history" - await processor.process_result(_make_record_data(session_num=4)) + await processor.process_record(_make_record_data(session_num=4)) assert processor._task_total["history"] == 1 assert processor._task_total.get("algebra", 0) == 0 - async def test_process_result_last_valid_session_num_succeeds(self) -> None: - processor = _make_results_processor() + async def test_process_record_last_valid_session_num_succeeds(self) -> None: + processor = _make_accuracy_accumulator() processor._tasks = ["test_task", "test_task"] - await processor.process_result(_make_record_data(session_num=1, correct=1.0)) + await processor.process_record(_make_record_data(session_num=1, correct=1.0)) assert processor._overall_total == 1 assert processor._overall_correct == 1 assert processor._task_correct["test_task"] == 1 - async def test_process_result_raises_if_not_configured(self) -> None: - """process_result must raise if on_dataset_configured was never called.""" - processor = _make_results_processor() + async def test_process_record_raises_if_not_configured(self) -> None: + """process_record must raise if on_dataset_configured was never called.""" + processor = _make_accuracy_accumulator() with pytest.raises(RuntimeError, match="dataset not configured"): - await processor.process_result(_make_record_data(session_num=0)) + await processor.process_record(_make_record_data(session_num=0)) - async def test_process_result_increments_overall_unparsed(self) -> None: - processor = _make_results_processor() + async def test_process_record_increments_overall_unparsed(self) -> None: + processor = _make_accuracy_accumulator() processor._tasks = ["algebra"] - await processor.process_result( + await processor.process_record( _make_record_data(session_num=0, correct=1.0, unparsed=1.0) ) assert processor._overall_unparsed == 1 assert processor._overall_total == 1 - async def test_process_result_increments_task_unparsed(self) -> None: - processor = _make_results_processor() + async def test_process_record_increments_task_unparsed(self) -> None: + processor = _make_accuracy_accumulator() processor._tasks = ["algebra"] - await processor.process_result( + await processor.process_record( _make_record_data(session_num=0, correct=0.0, unparsed=1.0) ) assert processor._task_unparsed["algebra"] == 1 - async def test_process_result_does_not_increment_unparsed_when_conforming( + async def test_process_record_does_not_increment_unparsed_when_conforming( self, ) -> None: - processor = _make_results_processor() + processor = _make_accuracy_accumulator() processor._tasks = ["algebra"] - await processor.process_result( + await processor.process_record( _make_record_data(session_num=0, correct=1.0, unparsed=0.0) ) assert processor._overall_unparsed == 0 assert processor._task_unparsed.get("algebra", 0) == 0 - async def test_process_result_missing_unparsed_key_treated_as_conforming( + async def test_process_record_missing_unparsed_key_treated_as_conforming( self, ) -> None: - """Records without accuracy.unparsed (e.g. from older graders) count as conforming.""" - processor = _make_results_processor() + """Records without accuracy_unparsed (e.g. from older graders) count as conforming.""" + processor = _make_accuracy_accumulator() processor._tasks = ["algebra"] data = MetricRecordsData( metadata=create_metric_metadata(session_num=0), - metrics={"accuracy.correct": 1.0}, # no accuracy.unparsed key + metrics={"accuracy_correct": 1.0}, # no accuracy_unparsed key ) - await processor.process_result(data) + await processor.process_record(data) assert processor._overall_unparsed == 0 diff --git a/tests/unit/common/models/test_record_models.py b/tests/unit/common/models/test_record_models.py index 88be1a3c1b..1e97641e82 100644 --- a/tests/unit/common/models/test_record_models.py +++ b/tests/unit/common/models/test_record_models.py @@ -5,15 +5,20 @@ from pydantic import BaseModel, Field, SerializeAsAny from aiperf.common.enums import SSEFieldType -from aiperf.common.models import MetricResult, ProfileResults, SSEMessage +from aiperf.common.models import ( + MetricResult, + ProfileResults, + SSEMessage, + TimesliceResult, +) from aiperf.common.models.export_models import JsonMetricResult class TestProfileResults: """Test cases for ProfileResults model.""" - def test_profile_results_with_timeslice_metric_results(self): - """Test ProfileResults can store timeslice metric results.""" + def test_profile_results_with_timeslices(self): + """Test ProfileResults stores accumulator-backed timeslices.""" metric_result = MetricResult( tag="request_latency", header="Request Latency", @@ -21,28 +26,40 @@ def test_profile_results_with_timeslice_metric_results(self): avg=100.0, count=10, ) - - timeslice_results = { - 0: [metric_result], - 1: [metric_result], - } + timeslices = [ + TimesliceResult( + start_ns=1_000_000_000, + end_ns=2_000_000_000, + metric_results=[metric_result], + ), + TimesliceResult( + start_ns=2_000_000_000, + end_ns=3_000_000_000, + metric_results=[metric_result], + ), + ] profile_results = ProfileResults( records=[metric_result], - timeslice_metric_results=timeslice_results, + timeslices=timeslices, completed=1, - start_ns=1000000000, - end_ns=2000000000, + start_ns=1_000_000_000, + end_ns=3_000_000_000, ) - assert profile_results.timeslice_metric_results is not None - assert 0 in profile_results.timeslice_metric_results - assert 1 in profile_results.timeslice_metric_results - assert len(profile_results.timeslice_metric_results[0]) == 1 - assert len(profile_results.timeslice_metric_results[1]) == 1 + assert profile_results.timeslices is not None + assert len(profile_results.timeslices) == 2 + assert ( + profile_results.timeslices[0].metric_results["request_latency"] + is metric_result + ) + assert ( + profile_results.timeslices[1].metric_results["request_latency"] + is metric_result + ) - def test_profile_results_without_timeslice_metric_results(self): - """Test ProfileResults works without timeslice metric results.""" + def test_profile_results_without_timeslices(self): + """Test ProfileResults works without timeslice results.""" metric_result = MetricResult( tag="request_latency", header="Request Latency", @@ -54,14 +71,14 @@ def test_profile_results_without_timeslice_metric_results(self): profile_results = ProfileResults( records=[metric_result], completed=1, - start_ns=1000000000, - end_ns=2000000000, + start_ns=1_000_000_000, + end_ns=2_000_000_000, ) - assert profile_results.timeslice_metric_results is None + assert profile_results.timeslices is None - def test_profile_results_with_empty_timeslice_dict(self): - """Test ProfileResults with empty timeslice results dict.""" + def test_profile_results_with_empty_timeslices(self): + """Test ProfileResults with empty timeslice list.""" metric_result = MetricResult( tag="request_latency", header="Request Latency", @@ -72,14 +89,13 @@ def test_profile_results_with_empty_timeslice_dict(self): profile_results = ProfileResults( records=[metric_result], - timeslice_metric_results={}, + timeslices=[], completed=1, - start_ns=1000000000, - end_ns=2000000000, + start_ns=1_000_000_000, + end_ns=2_000_000_000, ) - assert profile_results.timeslice_metric_results is not None - assert len(profile_results.timeslice_metric_results) == 0 + assert profile_results.timeslices == [] def test_profile_results_with_multiple_timeslices_and_metrics(self): """Test ProfileResults with multiple timeslices containing multiple metrics.""" @@ -99,25 +115,30 @@ def test_profile_results_with_multiple_timeslices_and_metrics(self): count=1, ) - timeslice_results = { - 0: [latency_result, throughput_result], - 1: [latency_result, throughput_result], - 2: [latency_result, throughput_result], - } + timeslices = [ + TimesliceResult( + start_ns=1_000_000_000 + i * 1_000_000_000, + end_ns=2_000_000_000 + i * 1_000_000_000, + metric_results=[latency_result, throughput_result], + ) + for i in range(3) + ] profile_results = ProfileResults( records=[latency_result, throughput_result], - timeslice_metric_results=timeslice_results, + timeslices=timeslices, completed=2, - start_ns=1000000000, - end_ns=3000000000, + start_ns=1_000_000_000, + end_ns=4_000_000_000, ) - assert profile_results.timeslice_metric_results is not None - assert len(profile_results.timeslice_metric_results) == 3 - for i in range(3): - assert i in profile_results.timeslice_metric_results - assert len(profile_results.timeslice_metric_results[i]) == 2 + assert profile_results.timeslices is not None + assert len(profile_results.timeslices) == 3 + for timeslice in profile_results.timeslices: + assert set(timeslice.metric_results) == { + "request_latency", + "request_throughput", + } class TestSSEMessageDataclass: diff --git a/tests/unit/exporters/conftest.py b/tests/unit/exporters/conftest.py index 027ec7cd5f..9220bacc15 100644 --- a/tests/unit/exporters/conftest.py +++ b/tests/unit/exporters/conftest.py @@ -9,7 +9,6 @@ import pytest from aiperf.common.enums import PrometheusMetricType -from aiperf.common.models import MetricResult from aiperf.common.models.export_models import ( EndpointData, GpuSummary, @@ -284,99 +283,6 @@ def empty_telemetry_results(): ) -@pytest.fixture -def sample_timeslice_metric_results(): - """Create sample timeslice metric results for testing.""" - return { - 0: [ - MetricResult( - tag="time_to_first_token", - header="Time to First Token", - unit="ms", - avg=45.2, - min=12.1, - max=89.3, - p50=44.0, - p90=78.0, - p99=88.0, - std=15.2, - ), - MetricResult( - tag="inter_token_latency", - header="Inter Token Latency", - unit="ms", - avg=5.1, - min=2.3, - max=12.4, - p50=4.8, - p90=9.2, - p99=11.8, - std=2.1, - ), - ], - 1: [ - MetricResult( - tag="time_to_first_token", - header="Time to First Token", - unit="ms", - avg=48.5, - min=15.2, - max=92.1, - p50=47.3, - p90=82.4, - p99=90.5, - std=16.1, - ), - MetricResult( - tag="inter_token_latency", - header="Inter Token Latency", - unit="ms", - avg=5.4, - min=2.5, - max=13.1, - p50=5.1, - p90=9.8, - p99=12.3, - std=2.3, - ), - ], - } - - -@pytest.fixture -def mock_results_with_timeslices(sample_timeslice_metric_results): - """Create mock results with timeslice data.""" - - class MockResultsWithTimeslices: - def __init__(self): - self.timeslice_metric_results = sample_timeslice_metric_results - self.records = [] - self.start_ns = None - self.end_ns = None - self.has_results = True - self.was_cancelled = False - self.error_summary = [] - - return MockResultsWithTimeslices() - - -@pytest.fixture -def mock_results_without_timeslices(): - """Create mock results without timeslice data.""" - - class MockResultsNoTimeslices: - def __init__(self): - self.timeslice_metric_results = None - self.records = [] - self.start_ns = None - self.end_ns = None - self.has_results = False - self.was_cancelled = False - self.error_summary = [] - - return MockResultsNoTimeslices() - - @pytest.fixture def sample_server_metrics_results(): """Create a sample ServerMetricsResults with realistic multi-endpoint data. diff --git a/tests/unit/gpu_telemetry/test_telemetry_integration.py b/tests/unit/gpu_telemetry/test_telemetry_integration.py index b379c0ac3e..4bc85c0e75 100644 --- a/tests/unit/gpu_telemetry/test_telemetry_integration.py +++ b/tests/unit/gpu_telemetry/test_telemetry_integration.py @@ -269,12 +269,12 @@ async def test_callback_pipeline_error_handling( # Mock the process_telemetry_record method to raise an exception original_process = faulty_processor.process_telemetry_record - async def failing_process_result(record): + async def failing_process_telemetry_record(record): if record.gpu_index == 0: # Fail on first GPU only raise ValueError("Simulated processing error") return await original_process(record) - faulty_processor.process_telemetry_record = failing_process_result + faulty_processor.process_telemetry_record = failing_process_telemetry_record async def failing_record_callback(records, collector_id): """Async callback that processes records and may encounter errors.""" diff --git a/tests/unit/metrics/test_list_metric_aggregation.py b/tests/unit/metrics/test_list_metric_aggregation.py index a99d582015..1ae83b888b 100644 --- a/tests/unit/metrics/test_list_metric_aggregation.py +++ b/tests/unit/metrics/test_list_metric_aggregation.py @@ -198,8 +198,8 @@ def test_welford_std_is_stable_on_large_offset_distribution(self) -> None: def test_protocol_runtime_isinstance(self) -> None: """Aggregator should satisfy the ``MetricAggregator`` protocol so - ``isinstance`` dispatch in ``MetricResultsProcessor`` and - ``DerivedSumMetric`` accepts both this and ``MetricArray``.""" + ``MetricsAccumulator`` and ``DerivedSumMetric`` accept both this and + ``MetricArray``.""" digest_agg = TDigestListMetricAggregator() array_agg = MetricArray() assert isinstance(digest_agg, MetricAggregator) diff --git a/tests/unit/metrics/test_power_efficiency_metrics.py b/tests/unit/metrics/test_power_efficiency_metrics.py index 6c98d9c673..aab2869faf 100644 --- a/tests/unit/metrics/test_power_efficiency_metrics.py +++ b/tests/unit/metrics/test_power_efficiency_metrics.py @@ -18,8 +18,8 @@ class TestPowerEfficiencyDeriveValueContract: The three power-efficiency classes inherit `BaseDerivedMetric` for registry integration but their values are produced by - `GPUTelemetryAccumulator.compute_efficiency_metrics`, not by the derivation - walk in `MetricResultsProcessor.update_derived_metrics`. Calling + `GPUTelemetryAccumulator.compute_efficiency_metrics`, not by the metrics + accumulator's derived-metric pass. Calling `_derive_value` directly must raise `NoMetricValue` with a message that names the tag, the operation, and the injection site — so a future contributor copy-pasting this as the "derived metric pattern" sees the diff --git a/tests/unit/network_latency/test_jsonl_writer.py b/tests/unit/network_latency/test_jsonl_writer.py index eac7b78d6c..b49befbdd8 100644 --- a/tests/unit/network_latency/test_jsonl_writer.py +++ b/tests/unit/network_latency/test_jsonl_writer.py @@ -91,7 +91,7 @@ async def test_process_single_sample_writes_one_line( ) async with aiperf_lifecycle(processor): - await processor.process_network_latency_sample(_sample()) + await processor.process_record(_sample()) output_file = ( cfg_probing_enabled.cfg.artifacts.network_latency_export_jsonl_file @@ -116,9 +116,7 @@ async def test_process_multiple_samples_writes_multiple_lines( async with aiperf_lifecycle(processor): for i in range(5): - await processor.process_network_latency_sample( - _sample(rtt_ns=1_000 + i) - ) + await processor.process_record(_sample(rtt_ns=1_000 + i)) output_file = ( cfg_probing_enabled.cfg.artifacts.network_latency_export_jsonl_file diff --git a/tests/unit/post_processors/conftest.py b/tests/unit/post_processors/conftest.py index b21e3cedf3..34655ba2e5 100644 --- a/tests/unit/post_processors/conftest.py +++ b/tests/unit/post_processors/conftest.py @@ -52,7 +52,6 @@ from aiperf.metrics.base_record_metric import BaseRecordMetric from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.plugin.enums import EndpointType -from aiperf.post_processors.metric_results_processor import MetricResultsProcessor from aiperf.post_processors.raw_record_writer_processor import RawRecordWriterProcessor from tests.unit.conftest import ( DEFAULT_FIRST_RESPONSE_NS, @@ -598,25 +597,6 @@ def setup_mock_registry_sequences( return valid_tags, error_tags -def create_results_processor_with_metrics( - run, *metrics: type[BaseMetric] -) -> MetricResultsProcessor: - """Create a MetricResultsProcessor with pre-configured metrics. - - Args: - run: BenchmarkRun for the processor - metrics: list of metric classes - - Returns: - Configured MetricResultsProcessor instance - """ - - processor = MetricResultsProcessor(run) - processor._tags_to_types = {metric.tag: metric.type for metric in metrics} - processor._instances_map = {metric.tag: metric() for metric in metrics} - return processor - - def create_accumulator_with_metrics(run, *metrics: type[BaseMetric]): """Construct a :class:`MetricsAccumulator` pre-configured with ``metrics``. @@ -655,9 +635,7 @@ def mock_metric_registry(monkeypatch): monkeypatch.setattr( "aiperf.post_processors.base_metrics_processor.MetricRegistry", mock_registry ) - monkeypatch.setattr( - "aiperf.post_processors.metric_results_processor.MetricRegistry", mock_registry - ) + monkeypatch.setattr("aiperf.metrics.accumulator.MetricRegistry", mock_registry) monkeypatch.setattr("aiperf.metrics.display_units.MetricRegistry", mock_registry) return mock_registry diff --git a/tests/unit/post_processors/test_metric_results_processor.py b/tests/unit/post_processors/test_metric_results_processor.py index fb6cf5ccd3..3812de5dd7 100644 --- a/tests/unit/post_processors/test_metric_results_processor.py +++ b/tests/unit/post_processors/test_metric_results_processor.py @@ -1,427 +1,185 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from unittest.mock import Mock, patch +from __future__ import annotations + +from unittest.mock import patch import pytest -from aiperf.common.enums import MetricType +from aiperf.common.constants import NANOS_PER_SECOND from aiperf.common.exceptions import NoMetricValue -from aiperf.common.models import MetricResult -from aiperf.metrics.list_metric_aggregation import TDigestListMetricAggregator -from aiperf.metrics.metric_dicts import MetricArray, MetricResultsDict -from aiperf.metrics.types.credit_drop_latency_metric import CreditDropLatencyMetric +from aiperf.metrics.accumulator import MetricsAccumulator +from aiperf.metrics.accumulator_models import AccumulatorMetricsSummary +from aiperf.metrics.metric_dicts import MetricResultsDict +from aiperf.metrics.types.inter_chunk_latency_metric import InterChunkLatencyMetric +from aiperf.metrics.types.max_response_metric import MaxResponseTimestampMetric +from aiperf.metrics.types.min_request_metric import MinRequestTimestampMetric from aiperf.metrics.types.request_count_metric import RequestCountMetric from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric from aiperf.metrics.types.request_throughput_metric import RequestThroughputMetric -from aiperf.post_processors.metric_results_processor import MetricResultsProcessor from tests.unit.post_processors.conftest import create_metric_records_message -class TestMetricResultsProcessor: - """Test cases for MetricResultsProcessor.""" +class TestMetricsAccumulator: + """Tests for the accumulator-backed metric summary engine.""" - def test_initialization(self, mock_metric_registry: Mock, mock_run) -> None: - """Test processor initialization sets up necessary data structures.""" - processor = MetricResultsProcessor(mock_run) + def test_initialization(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) - assert isinstance(processor.derive_funcs, dict) - assert isinstance(processor._results, dict) - assert isinstance(processor._tags_to_types, dict) - assert isinstance(processor._instances_map, dict) - assert isinstance(processor._tags_to_aggregate_funcs, dict) + assert accumulator.record_count == 0 + assert accumulator.column_store.count == 0 + assert accumulator._network_rtt_ns is None + assert isinstance(accumulator._derive_funcs, dict) + assert isinstance(accumulator._tags_to_types, dict) + assert isinstance(accumulator._metric_classes, dict) @pytest.mark.asyncio - async def test_process_result_record_metric( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test processing result for record metric accumulates values in the array.""" - processor = MetricResultsProcessor(mock_run) - processor._tags_to_types = {"test_record": MetricType.RECORD} - - message = create_metric_records_message( - x_request_id="test-1", - results=[{"test_record": 42.0}], - ) - await processor.process_result(message.to_data()) + async def test_process_record_metric_accumulates_values(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + + for idx, value in enumerate([42_000_000.0, 84_000_000.0]): + message = create_metric_records_message( + x_request_id=f"test-{idx}", + request_start_ns=1_000_000_000 + idx, + request_end_ns=1_100_000_000 + idx, + results=[{RequestLatencyMetric.tag: value}], + ) + await accumulator.process_record(message.to_data()) - assert "test_record" in processor._results - assert isinstance(processor._results["test_record"], MetricArray) - assert list(processor._results["test_record"].data) == [42.0] + summary = await accumulator.summarize() + result = summary.results[RequestLatencyMetric.tag] - # New data should expand the array - message2 = create_metric_records_message( - x_request_id="test-2", - request_start_ns=1_000_000_001, - results=[{"test_record": 84.0}], - ) - await processor.process_result(message2.to_data()) - assert list(processor._results["test_record"].data) == [42.0, 84.0] + assert accumulator.record_count == 2 + assert result.unit == "ms" + assert result.count == 2 + assert result.avg == pytest.approx(63.0) + assert result.sum == pytest.approx(126.0) @pytest.mark.asyncio - async def test_process_result_record_metric_list_values( - self, mock_metric_registry: Mock, mock_run + async def test_process_record_list_metric_summarizes_distribution( + self, mock_run ) -> None: - """List-valued record metrics use the t-digest aggregator (not MetricArray). - - T-digest preserves count/sum/min/max exactly; percentiles are - approximate but irrelevant to this test (3 samples). - """ - processor = MetricResultsProcessor(mock_run) - processor._tags_to_types = {"test_record": MetricType.RECORD} - + accumulator = MetricsAccumulator(mock_run) message = create_metric_records_message( x_request_id="test-1", - results=[{"test_record": [10.0, 20.0, 30.0]}], + results=[ + { + InterChunkLatencyMetric.tag: [ + 10_000_000.0, + 20_000_000.0, + 30_000_000.0, + ] + } + ], ) - await processor.process_result(message.to_data()) - assert "test_record" in processor._results - assert isinstance( - processor._results["test_record"], TDigestListMetricAggregator - ) - # Stat-shape check (count/sum/min/max are bit-exact via side-channel). - result = processor._results["test_record"].to_result( - tag="test_record", header="Test Record", unit="ms" - ) + await accumulator.process_record(message.to_data()) + + summary = await accumulator.summarize() + result = summary.results[InterChunkLatencyMetric.tag] assert result.count == 3 assert result.sum == pytest.approx(60.0) assert result.min == pytest.approx(10.0) assert result.max == pytest.approx(30.0) @pytest.mark.asyncio - async def test_process_result_aggregate_metric( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test processing result for aggregate metric updates aggregated value.""" - processor = MetricResultsProcessor(mock_run) - processor._tags_to_types = {RequestCountMetric.tag: MetricType.AGGREGATE} - processor._instances_map = {RequestCountMetric.tag: RequestCountMetric()} + async def test_process_aggregate_metric_sums_values(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + + for idx, value in enumerate([5, 3]): + message = create_metric_records_message( + x_request_id=f"test-{idx}", + request_start_ns=1_000_000_000 + idx, + results=[{RequestCountMetric.tag: value}], + ) + await accumulator.process_record(message.to_data()) - # Process two values and ensure they are accumulated - message1 = create_metric_records_message( - x_request_id="test-1", - results=[{RequestCountMetric.tag: 5}], - ) - await processor.process_result(message1.to_data()) - assert processor._results[RequestCountMetric.tag] == 5 + summary = await accumulator.summarize() - message2 = create_metric_records_message( - x_request_id="test-2", - request_start_ns=1_000_000_001, - results=[{RequestCountMetric.tag: 3}], - ) - await processor.process_result(message2.to_data()) - assert processor._results[RequestCountMetric.tag] == 8 + assert summary.results[RequestCountMetric.tag].avg == 8 @pytest.mark.asyncio - async def test_update_derived_metrics( - self, mock_metric_registry: Mock, mock_run + async def test_derived_metrics_are_computed_from_accumulated_results( + self, mock_run ) -> None: - """Test derived metrics are computed correctly.""" - - def mock_derive_func(results_dict: MetricResultsDict): - return 100.0 - - processor = MetricResultsProcessor(mock_run) - processor.derive_funcs = {RequestThroughputMetric.tag: mock_derive_func} + accumulator = MetricsAccumulator(mock_run) + message = create_metric_records_message( + x_request_id="test-1", + results=[ + { + RequestCountMetric.tag: 100, + MinRequestTimestampMetric.tag: 0, + MaxResponseTimestampMetric.tag: 10 * NANOS_PER_SECOND, + } + ], + ) - await processor.update_derived_metrics() + await accumulator.process_record(message.to_data()) + summary = await accumulator.summarize() - assert processor._results[RequestThroughputMetric.tag] == 100.0 + assert summary.results[RequestThroughputMetric.tag].avg == pytest.approx(10.0) @pytest.mark.asyncio - async def test_update_derived_metrics_handles_no_metric_value( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test derived metrics gracefully handle NoMetricValue exceptions.""" + async def test_derived_metrics_ignore_no_metric_value(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) - def failing_derive_func(results_dict: MetricResultsDict): + def failing_derive_func(results_dict: MetricResultsDict) -> float: raise NoMetricValue("Cannot derive value") - processor = MetricResultsProcessor(mock_run) - processor.derive_funcs = {RequestThroughputMetric.tag: failing_derive_func} + accumulator._derive_funcs = {RequestThroughputMetric.tag: failing_derive_func} - with patch.object(processor, "debug") as mock_debug: - await processor.update_derived_metrics() + with patch.object(accumulator, "debug") as mock_debug: + summary = await accumulator.summarize() - assert RequestThroughputMetric.tag not in processor._results - mock_debug.assert_called_once() + assert RequestThroughputMetric.tag not in summary.results + assert any( + "No metric value for derived metric" in str(call.args[0]) + for call in mock_debug.call_args_list + ) @pytest.mark.asyncio - async def test_update_derived_metrics_handles_value_error_exception( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test derived metrics gracefully handle ValueError exceptions.""" + async def test_derived_metrics_warn_on_unexpected_exception(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) - def failing_derive_func(results_dict: MetricResultsDict): + def failing_derive_func(results_dict: MetricResultsDict) -> float: raise ValueError("Calculation error") - processor = MetricResultsProcessor(mock_run) - processor.derive_funcs = {RequestThroughputMetric.tag: failing_derive_func} - - with patch.object(processor, "warning") as mock_warning: - await processor.update_derived_metrics() - - assert RequestThroughputMetric.tag not in processor._results - mock_warning.assert_called_once() - - @pytest.mark.asyncio - async def test_summarize(self, mock_metric_registry: Mock, mock_run) -> None: - """Test summarize returns list of MetricResult objects in display units. - - RequestLatencyMetric has unit=ns and display_unit=ms, so nanosecond - values should be converted to milliseconds in the output. - """ - mock_metric_registry.get_class.return_value = RequestLatencyMetric - - processor = MetricResultsProcessor(mock_run) - processor._tags_to_types = {RequestLatencyMetric.tag: MetricType.RECORD} - processor._instances_map = {RequestLatencyMetric.tag: RequestLatencyMetric()} - - processor._results[RequestLatencyMetric.tag] = MetricArray() - processor._results[RequestLatencyMetric.tag].append(42_000_000.0) + accumulator._derive_funcs = {RequestThroughputMetric.tag: failing_derive_func} - results = await processor.summarize() + with patch.object(accumulator, "warning") as mock_warning: + summary = await accumulator.summarize() - assert len(results) == 1 - assert isinstance(results[0], MetricResult) - assert results[0].tag == RequestLatencyMetric.tag - assert results[0].unit == "ms" - assert results[0].avg == 42.0 + assert RequestThroughputMetric.tag not in summary.results + mock_warning.assert_called_once() @pytest.mark.asyncio - async def test_full_metrics(self, mock_metric_registry: Mock, mock_run) -> None: - """Test full_metrics returns the complete results dict including derived metrics.""" - - def mock_derive_func(results_dict: MetricResultsDict): - return 200.0 - - processor = MetricResultsProcessor(mock_run) - processor.derive_funcs = {RequestThroughputMetric.tag: mock_derive_func} - processor._results["base_metric"] = 100.0 - - full_results = await processor.full_metrics() - - assert "base_metric" in full_results - assert RequestThroughputMetric.tag in full_results - assert full_results["base_metric"] == 100.0 - assert full_results[RequestThroughputMetric.tag] == 200.0 - - def test_create_metric_result_from_scalar( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test creating MetricResult from scalar value.""" - processor = MetricResultsProcessor(mock_run) - processor._instances_map = {RequestLatencyMetric.tag: RequestLatencyMetric()} - - result = processor._create_metric_result(RequestLatencyMetric.tag, 42) - - assert isinstance(result, MetricResult) - assert result.tag == RequestLatencyMetric.tag - assert result.header == RequestLatencyMetric.header - assert result.unit == str(RequestLatencyMetric.unit) - assert result.avg == 42 - assert result.count == 1 - - def test_create_metric_result_from_metric_array( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test creating MetricResult from MetricArray.""" - processor = MetricResultsProcessor(mock_run) - processor._instances_map = {RequestLatencyMetric.tag: RequestLatencyMetric()} - metric_array = MetricArray() - metric_array.extend([10.0, 20.0, 30.0]) - - expected_result = MetricResult( - tag=RequestLatencyMetric.tag, - header=RequestLatencyMetric.header, - unit=str(RequestLatencyMetric.unit), - avg=20.0, - count=3, - ) - metric_array.to_result = Mock(return_value=expected_result) - - result = processor._create_metric_result(RequestLatencyMetric.tag, metric_array) - - assert result == expected_result - metric_array.to_result.assert_called_once_with( - RequestLatencyMetric.tag, - RequestLatencyMetric.header, - str(RequestLatencyMetric.unit), + async def test_summarize_returns_typed_summary(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + await accumulator.process_record( + create_metric_records_message( + x_request_id="test-1", + results=[{RequestLatencyMetric.tag: 42_000_000.0}], + ).to_data() ) - def test_create_metric_result_invalid_type( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test creating MetricResult with invalid value type raises a ValueError.""" - processor = MetricResultsProcessor(mock_run) - - processor._instances_map = {RequestLatencyMetric.tag: RequestLatencyMetric()} - with pytest.raises(ValueError, match="Unexpected values type"): - processor._create_metric_result( - RequestLatencyMetric.tag, {"invalid": "dict"} - ) - - @pytest.mark.asyncio - async def test_get_instances_map_default_behavior( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test default get_instances_map returns shared instances map regardless of request_start_ns.""" - processor = MetricResultsProcessor(mock_run) - - # Set up a metric - processor._instances_map = {RequestCountMetric.tag: RequestCountMetric()} - - # Call with None (should be ignored in base implementation) - instances_map_none = await processor.get_instances_map(None) - assert instances_map_none is processor._instances_map - - # Call with a timestamp (should also be ignored in base implementation) - instances_map_with_time = await processor.get_instances_map(1000000000) - assert instances_map_with_time is processor._instances_map + summary = await accumulator.summarize() - # Both should return the same shared instances map - assert instances_map_none is instances_map_with_time + assert isinstance(summary, AccumulatorMetricsSummary) + assert summary.results[RequestLatencyMetric.tag].unit == "ms" + assert summary.results[RequestLatencyMetric.tag].avg == pytest.approx(42.0) @pytest.mark.asyncio - async def test_get_results_default_behavior( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test default get_results returns shared results dict regardless of request_start_ns.""" - processor = MetricResultsProcessor(mock_run) - - # Set up some results - processor._results["test_metric"] = 42 - - # Call with None (should be ignored in base implementation) - results_dict_none = await processor.get_results(None) - assert results_dict_none is processor._results - assert results_dict_none["test_metric"] == 42 - - # Call with a timestamp (should also be ignored in base implementation) - results_dict_with_time = await processor.get_results(1000000000) - assert results_dict_with_time is processor._results - assert results_dict_with_time["test_metric"] == 42 - - # Both should return the same shared results dict - assert results_dict_none is results_dict_with_time - - -class TestShouldIncludeInSummary: - """Tests for _should_include_in_summary() filtering logic. - - Uses real BaseRecordMetric subclasses with actual MetricFlags to test - filtering behavior. The mock_metric_registry fixture intercepts - __init_subclass__ registration, so no cleanup is needed. - """ - - def test_unknown_tag_raises_key_error( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Unknown tags (not in _instances_map) raise KeyError.""" - processor = MetricResultsProcessor(mock_run) - processor._instances_map = {} - - with pytest.raises(KeyError): - processor._should_include_in_summary("nonexistent_tag") - - def test_public_metric_included(self, mock_metric_registry: Mock, mock_run) -> None: - """Metrics with no special flags are always included.""" - processor = MetricResultsProcessor(mock_run) - processor._instances_map = {RequestLatencyMetric.tag: RequestLatencyMetric()} - - assert processor._should_include_in_summary(RequestLatencyMetric.tag) is True - - def test_internal_metric_excluded_by_default( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """INTERNAL metrics are excluded when SHOW_INTERNAL_METRICS is False.""" - processor = MetricResultsProcessor(mock_run) - processor._instances_map = { - CreditDropLatencyMetric.tag: CreditDropLatencyMetric() - } - - with patch( - "aiperf.post_processors.metric_results_processor.Environment.DEV" - ) as mock_dev: - mock_dev.SHOW_INTERNAL_METRICS = False - mock_dev.SHOW_EXPERIMENTAL_METRICS = False - - assert ( - processor._should_include_in_summary(CreditDropLatencyMetric.tag) - is False - ) + async def test_full_metrics_returns_raw_metric_results(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + await accumulator.process_record( + create_metric_records_message( + x_request_id="test-1", + results=[{RequestLatencyMetric.tag: 42_000_000.0}], + ).to_data() + ) - def test_internal_metric_included_when_flag_enabled( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """INTERNAL metrics are included when SHOW_INTERNAL_METRICS is True.""" - processor = MetricResultsProcessor(mock_run) - processor._instances_map = { - CreditDropLatencyMetric.tag: CreditDropLatencyMetric() - } - - with patch( - "aiperf.post_processors.metric_results_processor.Environment.DEV" - ) as mock_dev: - mock_dev.SHOW_INTERNAL_METRICS = True - mock_dev.SHOW_EXPERIMENTAL_METRICS = False - - assert ( - processor._should_include_in_summary(CreditDropLatencyMetric.tag) - is True - ) + full_results = await accumulator.full_metrics() - @pytest.mark.parametrize( - ("show_experimental", "expected"), - [ - (False, False), - (True, True), - ], - ids=["excluded_by_default", "included_when_enabled"], - ) - def test_experimental_metric_filtering( - self, - mock_metric_registry: Mock, - mock_run, - experimental_metric_cls, - show_experimental: bool, - expected: bool, - ) -> None: - """EXPERIMENTAL metrics respect the SHOW_EXPERIMENTAL_METRICS flag.""" - processor = MetricResultsProcessor(mock_run) - processor._instances_map = { - experimental_metric_cls.tag: experimental_metric_cls() - } - - with patch( - "aiperf.post_processors.metric_results_processor.Environment.DEV" - ) as mock_dev: - mock_dev.SHOW_INTERNAL_METRICS = False - mock_dev.SHOW_EXPERIMENTAL_METRICS = show_experimental - - assert ( - processor._should_include_in_summary(experimental_metric_cls.tag) - is expected - ) - - def test_internal_and_experimental_metric_excluded_when_both_disabled( - self, - mock_metric_registry: Mock, - mock_run, - dual_flag_metric_cls, - ) -> None: - """Metrics with both INTERNAL and EXPERIMENTAL flags are excluded when both flags are disabled.""" - processor = MetricResultsProcessor(mock_run) - processor._instances_map = {dual_flag_metric_cls.tag: dual_flag_metric_cls()} - - with patch( - "aiperf.post_processors.metric_results_processor.Environment.DEV" - ) as mock_dev: - mock_dev.SHOW_INTERNAL_METRICS = False - mock_dev.SHOW_EXPERIMENTAL_METRICS = False - - assert ( - processor._should_include_in_summary(dual_flag_metric_cls.tag) is False - ) + assert RequestLatencyMetric.tag in full_results + assert full_results[RequestLatencyMetric.tag].avg == pytest.approx(42_000_000.0) diff --git a/tests/unit/post_processors/test_metrics_accumulator.py b/tests/unit/post_processors/test_metrics_accumulator.py index ac49e8fc28..d92644496d 100644 --- a/tests/unit/post_processors/test_metrics_accumulator.py +++ b/tests/unit/post_processors/test_metrics_accumulator.py @@ -670,7 +670,7 @@ async def test_timeslice_max_aggregate( processor = MetricsAccumulator(mock_run) processor._tags_to_types = {"max_ts": MetricType.AGGREGATE} processor._aggregation_kinds = {"max_ts": AggregationKind.MAX} - processor._metric_classes = {"max_ts": RequestLatencyMetric} + processor._metric_classes = {"max_ts": RequestCountMetric} msg1 = create_metric_records_message( x_request_id="test-1", @@ -702,7 +702,7 @@ async def test_timeslice_min_aggregate( processor = MetricsAccumulator(mock_run) processor._tags_to_types = {"min_ts": MetricType.AGGREGATE} processor._aggregation_kinds = {"min_ts": AggregationKind.MIN} - processor._metric_classes = {"min_ts": RequestLatencyMetric} + processor._metric_classes = {"min_ts": RequestCountMetric} msg1 = create_metric_records_message( x_request_id="test-1", diff --git a/tests/unit/post_processors/test_network_adjusted_metrics.py b/tests/unit/post_processors/test_network_adjusted_metrics.py index 3d941c7941..51218739a0 100644 --- a/tests/unit/post_processors/test_network_adjusted_metrics.py +++ b/tests/unit/post_processors/test_network_adjusted_metrics.py @@ -1,12 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for the network-RTT-adjusted metric injection in MetricResultsProcessor. - -These tests exercise the real MetricRegistry (not the mocked one) so that the -registered ``network_adjusted_*`` / ``network_rtt`` classes resolve and flow -through the normal ``_create_metric_result`` / ``to_display_unit`` path. -""" +"""Tests for network-RTT-adjusted metrics injected by MetricsAccumulator.""" from __future__ import annotations @@ -14,7 +9,7 @@ import pytest from aiperf.common.models import MetricResult -from aiperf.metrics.metric_dicts import MetricArray +from aiperf.metrics.accumulator import MetricsAccumulator from aiperf.metrics.metric_registry import MetricRegistry from aiperf.metrics.types.network_adjusted_metrics import ( NETWORK_ADJUSTED_SOURCES, @@ -25,7 +20,7 @@ TimeToFirstOutputTokenMetric, ) from aiperf.metrics.types.ttft_metric import TTFTMetric -from aiperf.post_processors.metric_results_processor import MetricResultsProcessor +from tests.unit.post_processors.conftest import create_metric_records_message _PERCENTILE_ATTRS = ("p1", "p5", "p10", "p25", "p50", "p75", "p90", "p95", "p99") _SHIFT_ATTRS = ("min", "max", "avg", *_PERCENTILE_ATTRS) @@ -45,15 +40,26 @@ ] -def _seed_array(processor: MetricResultsProcessor, tag: str, values_ns) -> None: - """Seed a MetricArray of nanosecond values into the processor results dict.""" - array = MetricArray() - array.extend(values_ns) - processor._results[tag] = array +async def _seed_records( + accumulator: MetricsAccumulator, + series_by_tag: dict[str, list[float]], +) -> None: + """Seed equal-length per-record metric series into the accumulator.""" + length = len(next(iter(series_by_tag.values()))) + for idx in range(length): + metrics = {tag: values[idx] for tag, values in series_by_tag.items()} + await accumulator.process_record( + create_metric_records_message( + x_request_id=f"r-{idx}", + request_start_ns=1_000_000_000 + idx, + request_end_ns=1_100_000_000 + idx, + results=[metrics], + ).to_data() + ) -def _results_by_tag(results: list[MetricResult]) -> dict[str, MetricResult]: - return {r.tag: r for r in results} +def _results_by_tag(results: dict[str, MetricResult]) -> dict[str, MetricResult]: + return results class TestNetworkAdjustedShift: @@ -64,11 +70,13 @@ async def test_adjusted_request_latency_every_stat_shifts_by_rtt( self, mock_run ) -> None: rtt_ns = 500_000.0 # 0.5 ms, below the minimum sample so no clamping - processor = MetricResultsProcessor(mock_run) - _seed_array(processor, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) - processor.set_network_rtt_ns(rtt_ns) + accumulator = MetricsAccumulator(mock_run) + await _seed_records( + accumulator, {RequestLatencyMetric.tag: _REQUEST_LATENCY_NS} + ) + accumulator.set_network_rtt_ns(rtt_ns) - results = _results_by_tag(await processor.summarize()) + results = _results_by_tag((await accumulator.summarize()).results) raw = results[RequestLatencyMetric.tag] adjusted = results["network_adjusted_request_latency"] @@ -82,11 +90,13 @@ async def test_adjusted_request_latency_every_stat_shifts_by_rtt( @pytest.mark.asyncio async def test_adjusted_request_latency_std_unchanged(self, mock_run) -> None: rtt_ns = 500_000.0 - processor = MetricResultsProcessor(mock_run) - _seed_array(processor, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) - processor.set_network_rtt_ns(rtt_ns) + accumulator = MetricsAccumulator(mock_run) + await _seed_records( + accumulator, {RequestLatencyMetric.tag: _REQUEST_LATENCY_NS} + ) + accumulator.set_network_rtt_ns(rtt_ns) - results = _results_by_tag(await processor.summarize()) + results = _results_by_tag((await accumulator.summarize()).results) assert results["network_adjusted_request_latency"].std == pytest.approx( results[RequestLatencyMetric.tag].std @@ -94,11 +104,13 @@ async def test_adjusted_request_latency_std_unchanged(self, mock_run) -> None: @pytest.mark.asyncio async def test_adjusted_count_matches_source(self, mock_run) -> None: - processor = MetricResultsProcessor(mock_run) - _seed_array(processor, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) - processor.set_network_rtt_ns(500_000.0) + accumulator = MetricsAccumulator(mock_run) + await _seed_records( + accumulator, {RequestLatencyMetric.tag: _REQUEST_LATENCY_NS} + ) + accumulator.set_network_rtt_ns(500_000.0) - results = _results_by_tag(await processor.summarize()) + results = _results_by_tag((await accumulator.summarize()).results) assert ( results["network_adjusted_request_latency"].count @@ -114,11 +126,13 @@ class TestNetworkAdjustedClamp: async def test_rtt_exceeds_some_samples_floors_at_zero(self, mock_run) -> None: # RTT of 3.5 ms exceeds the three smallest samples (1, 2, 3 ms). rtt_ns = 3_500_000.0 - processor = MetricResultsProcessor(mock_run) - _seed_array(processor, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) - processor.set_network_rtt_ns(rtt_ns) + accumulator = MetricsAccumulator(mock_run) + await _seed_records( + accumulator, {RequestLatencyMetric.tag: _REQUEST_LATENCY_NS} + ) + accumulator.set_network_rtt_ns(rtt_ns) - results = _results_by_tag(await processor.summarize()) + results = _results_by_tag((await accumulator.summarize()).results) adjusted = results["network_adjusted_request_latency"] for attr in _SHIFT_ATTRS: @@ -134,12 +148,17 @@ class TestNetworkAdjustedInterTokenInvariance: async def test_no_network_adjusted_inter_token_latency_tag_emitted( self, mock_run ) -> None: - processor = MetricResultsProcessor(mock_run) - _seed_array(processor, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) - _seed_array(processor, TTFTMetric.tag, [v / 2 for v in _REQUEST_LATENCY_NS]) - processor.set_network_rtt_ns(500_000.0) + accumulator = MetricsAccumulator(mock_run) + await _seed_records( + accumulator, + { + RequestLatencyMetric.tag: _REQUEST_LATENCY_NS, + TTFTMetric.tag: [v / 2 for v in _REQUEST_LATENCY_NS], + }, + ) + accumulator.set_network_rtt_ns(500_000.0) - tags = {r.tag for r in await processor.summarize()} + tags = set((await accumulator.summarize()).results) assert "network_adjusted_inter_token_latency" not in tags assert "network_adjusted_inter_chunk_latency" not in tags @@ -149,12 +168,17 @@ async def test_itl_algebraically_cancels_rtt(self, mock_run) -> None: """ITL = request_latency - ttft, so the subtracted RTT cancels exactly.""" rtt_ns = 500_000.0 ttft_ns = [v / 2 for v in _REQUEST_LATENCY_NS] - processor = MetricResultsProcessor(mock_run) - _seed_array(processor, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) - _seed_array(processor, TTFTMetric.tag, ttft_ns) - processor.set_network_rtt_ns(rtt_ns) + accumulator = MetricsAccumulator(mock_run) + await _seed_records( + accumulator, + { + RequestLatencyMetric.tag: _REQUEST_LATENCY_NS, + TTFTMetric.tag: ttft_ns, + }, + ) + accumulator.set_network_rtt_ns(rtt_ns) - results = _results_by_tag(await processor.summarize()) + results = _results_by_tag((await accumulator.summarize()).results) adj_rl = results["network_adjusted_request_latency"].avg adj_ttft = results["network_adjusted_time_to_first_token"].avg @@ -171,16 +195,26 @@ class TestNetworkAdjustedNonDestructive: async def test_raw_metrics_identical_with_and_without_rtt(self, mock_run) -> None: ttft_ns = [v / 2 for v in _REQUEST_LATENCY_NS] - baseline = MetricResultsProcessor(mock_run) - _seed_array(baseline, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) - _seed_array(baseline, TTFTMetric.tag, ttft_ns) - baseline_results = _results_by_tag(await baseline.summarize()) - - adjusted = MetricResultsProcessor(mock_run) - _seed_array(adjusted, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) - _seed_array(adjusted, TTFTMetric.tag, ttft_ns) + baseline = MetricsAccumulator(mock_run) + await _seed_records( + baseline, + { + RequestLatencyMetric.tag: _REQUEST_LATENCY_NS, + TTFTMetric.tag: ttft_ns, + }, + ) + baseline_results = _results_by_tag((await baseline.summarize()).results) + + adjusted = MetricsAccumulator(mock_run) + await _seed_records( + adjusted, + { + RequestLatencyMetric.tag: _REQUEST_LATENCY_NS, + TTFTMetric.tag: ttft_ns, + }, + ) adjusted.set_network_rtt_ns(500_000.0) - adjusted_results = _results_by_tag(await adjusted.summarize()) + adjusted_results = _results_by_tag((await adjusted.summarize()).results) for tag in (RequestLatencyMetric.tag, TTFTMetric.tag): for attr in (*_SHIFT_ATTRS, "std", "count", "sum"): @@ -189,16 +223,18 @@ async def test_raw_metrics_identical_with_and_without_rtt(self, mock_run) -> Non ), f"raw {tag}.{attr} was mutated by RTT injection" @pytest.mark.asyncio - async def test_source_array_object_not_mutated_in_place(self, mock_run) -> None: - processor = MetricResultsProcessor(mock_run) - _seed_array(processor, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) - original = np.array(processor._results[RequestLatencyMetric.tag].data) - processor.set_network_rtt_ns(500_000.0) + async def test_source_column_not_mutated_in_place(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + await _seed_records( + accumulator, {RequestLatencyMetric.tag: _REQUEST_LATENCY_NS} + ) + original = accumulator.column_store.numeric(RequestLatencyMetric.tag).copy() + accumulator.set_network_rtt_ns(500_000.0) - await processor.summarize() + await accumulator.summarize() np.testing.assert_array_equal( - processor._results[RequestLatencyMetric.tag].data, original + accumulator.column_store.numeric(RequestLatencyMetric.tag), original ) @@ -207,21 +243,25 @@ class TestNetworkAdjustedNoOp: @pytest.mark.asyncio async def test_rtt_never_set_emits_no_adjusted_rows(self, mock_run) -> None: - processor = MetricResultsProcessor(mock_run) - _seed_array(processor, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) + accumulator = MetricsAccumulator(mock_run) + await _seed_records( + accumulator, {RequestLatencyMetric.tag: _REQUEST_LATENCY_NS} + ) - tags = {r.tag for r in await processor.summarize()} + tags = set((await accumulator.summarize()).results) assert not any(tag.startswith("network_adjusted_") for tag in tags) assert NetworkRttMetric.tag not in tags @pytest.mark.asyncio async def test_set_rtt_none_emits_no_adjusted_rows(self, mock_run) -> None: - processor = MetricResultsProcessor(mock_run) - _seed_array(processor, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) - processor.set_network_rtt_ns(None) + accumulator = MetricsAccumulator(mock_run) + await _seed_records( + accumulator, {RequestLatencyMetric.tag: _REQUEST_LATENCY_NS} + ) + accumulator.set_network_rtt_ns(None) - tags = {r.tag for r in await processor.summarize()} + tags = set((await accumulator.summarize()).results) assert not any(tag.startswith("network_adjusted_") for tag in tags) assert NetworkRttMetric.tag not in tags @@ -233,11 +273,13 @@ class TestNetworkRttSummary: @pytest.mark.asyncio async def test_network_rtt_row_present_with_avg_in_ms(self, mock_run) -> None: rtt_ns = 750_000.0 - processor = MetricResultsProcessor(mock_run) - _seed_array(processor, RequestLatencyMetric.tag, _REQUEST_LATENCY_NS) - processor.set_network_rtt_ns(rtt_ns) + accumulator = MetricsAccumulator(mock_run) + await _seed_records( + accumulator, {RequestLatencyMetric.tag: _REQUEST_LATENCY_NS} + ) + accumulator.set_network_rtt_ns(rtt_ns) - results = _results_by_tag(await processor.summarize()) + results = _results_by_tag((await accumulator.summarize()).results) assert NetworkRttMetric.tag in results net_rtt = results[NetworkRttMetric.tag] @@ -279,3 +321,86 @@ def test_network_adjusted_sources_map_to_registered_metrics(self) -> None: for adjusted_tag, source_tag in NETWORK_ADJUSTED_SOURCES.items(): assert adjusted_tag in MetricRegistry.all_tags() assert source_tag in MetricRegistry.all_tags() + + +@pytest.mark.asyncio +class TestNetworkAdjustedTimeslices: + """Network-adjusted metrics must be injected per timeslice window, not only + into the overall summary (parity with the overall ``summarize()`` path).""" + + async def _seed_two_windows(self, accumulator: MetricsAccumulator) -> None: + # 1-second slices; two record clusters ~2s apart land in distinct bins. + accumulator._slice_duration_ns = 1_000_000_000 + starts = [1_000_000_000, 1_000_500_000, 3_000_000_000, 3_000_500_000] + latencies = [4_000_000.0, 6_000_000.0, 4_000_000.0, 6_000_000.0] + for idx, (start, lat) in enumerate(zip(starts, latencies, strict=True)): + await accumulator.process_record( + create_metric_records_message( + x_request_id=f"r-{idx}", + request_start_ns=start, + request_end_ns=start + int(lat), + results=[{RequestLatencyMetric.tag: lat, TTFTMetric.tag: lat / 2}], + ).to_data() + ) + + async def test_network_adjusted_present_in_every_timeslice(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + await self._seed_two_windows(accumulator) + accumulator.set_network_rtt_ns(1_000_000.0) # 1ms + + summary = await accumulator.summarize() + + assert summary.timeslices is not None + assert len(summary.timeslices) >= 2 + for ts in summary.timeslices: + assert "network_adjusted_request_latency" in ts.metric_results, ( + f"timeslice [{ts.start_ns},{ts.end_ns}] missing " + "network_adjusted_request_latency" + ) + + async def test_timeslice_adjusted_shifts_by_rtt(self, mock_run) -> None: + # Per-window adjusted latency avg is the raw avg minus the RTT (both + # windows have avg = 5ms; RTT = 1ms -> adjusted avg = 4ms). + accumulator = MetricsAccumulator(mock_run) + await self._seed_two_windows(accumulator) + rtt_ns = 1_000_000.0 + accumulator.set_network_rtt_ns(rtt_ns) + + summary = await accumulator.summarize() + + assert summary.timeslices + for ts in summary.timeslices: + raw = ts.metric_results["request_latency"].avg + adjusted = ts.metric_results["network_adjusted_request_latency"].avg + # Values are in display units (ms); RTT converts to 1ms. + assert adjusted == pytest.approx(raw - 1.0) + + async def test_no_network_adjusted_in_timeslice_without_rtt(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + await self._seed_two_windows(accumulator) + # No set_network_rtt_ns -> no adjustment anywhere. + + summary = await accumulator.summarize() + + assert summary.timeslices + for ts in summary.timeslices: + assert "network_adjusted_request_latency" not in ts.metric_results + + async def test_overall_not_double_adjusted_when_timeslices_enabled( + self, mock_run + ) -> None: + # The per-window injection reads the pristine source array (never writes + # back to the store), so enabling timeslices must NOT double-subtract the + # RTT from the overall summary: overall adjusted == raw - 1x RTT, not 2x. + accumulator = MetricsAccumulator(mock_run) + await self._seed_two_windows(accumulator) # slice_duration set + accumulator.set_network_rtt_ns(1_000_000.0) # 1ms + + summary = await accumulator.summarize() + + assert summary.timeslices # per-window injection actually ran + raw_avg = summary.results["request_latency"].avg + adj_avg = summary.results["network_adjusted_request_latency"].avg + assert adj_avg == pytest.approx(raw_avg - 1.0) # exactly one RTT (1ms) + # And the raw source metric is untouched (5ms mean across both windows). + assert raw_avg == pytest.approx(5.0) diff --git a/tests/unit/post_processors/test_otel_error_paths.py b/tests/unit/post_processors/test_otel_error_paths.py index 8569b5b8bb..f584336051 100644 --- a/tests/unit/post_processors/test_otel_error_paths.py +++ b/tests/unit/post_processors/test_otel_error_paths.py @@ -152,10 +152,10 @@ async def test_drop_oldest_handles_exception( assert result is False @pytest.mark.asyncio - async def test_process_result_skips_when_not_ready( + async def test_process_record_skips_when_not_ready( self, mock_cfg: BenchmarkConfig ) -> None: - """process_result should silently return when streaming is not ready.""" + """process_record should silently return when streaming is not ready.""" with patch( "aiperf.post_processors.otel_metrics_results_processor.run_otel_streaming_fanout" ): @@ -166,7 +166,7 @@ async def test_process_result_skips_when_not_ready( processor._streaming_ready = False record = MagicMock() - await processor.process_result(record) + await processor.process_record(record) # No exception raised diff --git a/tests/unit/post_processors/test_otel_metrics_results_processor.py b/tests/unit/post_processors/test_otel_metrics_results_processor.py index ca02e6cb41..0e1da840f4 100644 --- a/tests/unit/post_processors/test_otel_metrics_results_processor.py +++ b/tests/unit/post_processors/test_otel_metrics_results_processor.py @@ -178,7 +178,7 @@ def test_init_otel_import_failure_falls_back_to_mlflow_only( assert processor._mlflow_live_enabled is True @pytest.mark.asyncio - async def test_process_result_records_histogram_values_by_metric( + async def test_process_record_records_histogram_values_by_metric( self, cfg_otel: BenchmarkConfig, ) -> None: @@ -197,7 +197,7 @@ async def test_process_result_records_histogram_values_by_metric( } ] ).to_data() - await processor.process_result(metric_record) + await processor.process_record(metric_record) histogram_events = [ e for e in fake_queue.events if e.get("type") == "histogram_record" @@ -212,7 +212,7 @@ async def test_process_result_records_histogram_values_by_metric( assert "attributes" in event["payload"] # type: ignore[operator] @pytest.mark.asyncio - async def test_process_result_skips_metrics_when_metrics_telemetry_disabled( + async def test_process_record_skips_metrics_when_metrics_telemetry_disabled( self, tmp_artifact_dir, ) -> None: @@ -236,7 +236,7 @@ async def test_process_result_skips_metrics_when_metrics_telemetry_disabled( metric_record = create_metric_records_message( results=[{"request_latency_ns": 123_000_000}] ).to_data() - await processor.process_result(metric_record) + await processor.process_record(metric_record) histogram_events = [ e for e in fake_queue.events if e.get("type") == "histogram_record" @@ -244,7 +244,7 @@ async def test_process_result_skips_metrics_when_metrics_telemetry_disabled( assert histogram_events == [] @pytest.mark.asyncio - async def test_process_result_skips_timing_when_timing_telemetry_disabled( + async def test_process_record_skips_timing_when_timing_telemetry_disabled( self, tmp_artifact_dir, ) -> None: @@ -278,7 +278,7 @@ async def test_process_result_skips_timing_when_timing_telemetry_disabled( cancelled_sessions=0, total_session_turns=1, ) - await processor.process_result(timing_stats) + await processor.process_record(timing_stats) counter_events = [ e for e in fake_queue.events if e.get("type") == "counter_add" @@ -290,7 +290,7 @@ async def test_process_result_skips_timing_when_timing_telemetry_disabled( assert up_down_events == [] @pytest.mark.asyncio - async def test_process_result_records_timing_counters_and_gauge_like_metrics( + async def test_process_record_records_timing_counters_and_gauge_like_metrics( self, cfg_otel: BenchmarkConfig, ) -> None: @@ -316,7 +316,7 @@ async def test_process_result_records_timing_counters_and_gauge_like_metrics( grace_period_timeout_triggered=False, was_cancelled=False, ) - await processor.process_result(timing_stats) + await processor.process_record(timing_stats) counter_events = [ e for e in fake_queue.events if e.get("type") == "counter_add" @@ -353,7 +353,7 @@ async def test_process_result_records_timing_counters_and_gauge_like_metrics( assert "aiperf.timing.phase.was_cancelled" not in up_down_by_name @pytest.mark.asyncio - async def test_process_result_timing_uses_delta_values_for_cumulative_counters( + async def test_process_record_timing_uses_delta_values_for_cumulative_counters( self, cfg_otel: BenchmarkConfig, ) -> None: @@ -395,8 +395,8 @@ async def test_process_result_timing_uses_delta_values_for_cumulative_counters( grace_period_timeout_triggered=False, was_cancelled=False, ) - await processor.process_result(first_stats) - await processor.process_result(second_stats) + await processor.process_record(first_stats) + await processor.process_record(second_stats) counter_events = [ e for e in fake_queue.events if e.get("type") == "counter_add" @@ -547,7 +547,7 @@ def Queue(self, maxsize: int): # noqa: N802 assert processor._fanout_process is None @pytest.mark.asyncio - async def test_process_result_fanout_emits_metric_and_timing_events( + async def test_process_record_fanout_emits_metric_and_timing_events( self, cfg_otel_mlflow: BenchmarkConfig, ) -> None: @@ -560,7 +560,7 @@ async def test_process_result_fanout_emits_metric_and_timing_events( metric_record = create_metric_records_message( results=[{"request_latency_ns": 123_000_000, "request_count": 1}] ).to_data() - await processor.process_result(metric_record) + await processor.process_record(metric_record) timing_stats = CreditPhaseStats( phase=CreditPhase.PROFILING, @@ -575,7 +575,7 @@ async def test_process_result_fanout_emits_metric_and_timing_events( cancelled_sessions=0, total_session_turns=9, ) - await processor.process_result(timing_stats) + await processor.process_record(timing_stats) event_types = [str(event.get("type")) for event in fake_queue.events] assert "histogram_record" in event_types diff --git a/tests/unit/post_processors/test_post_processor_integration.py b/tests/unit/post_processors/test_post_processor_integration.py index d89ef5ea94..3102944d48 100644 --- a/tests/unit/post_processors/test_post_processor_integration.py +++ b/tests/unit/post_processors/test_post_processor_integration.py @@ -1,6 +1,6 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Integration unit tests for post-processing pipeline.""" +"""Integration unit tests for the metrics record/accumulator pipeline.""" from unittest.mock import Mock @@ -8,21 +8,21 @@ from aiperf.common.constants import NANOS_PER_SECOND from aiperf.common.models import ParsedResponseRecord -from aiperf.metrics.metric_dicts import MetricArray -from aiperf.metrics.types.benchmark_duration_metric import BenchmarkDurationMetric +from aiperf.metrics.accumulator import MetricsAccumulator +from aiperf.metrics.accumulator_models import AccumulatorMetricsSummary from aiperf.metrics.types.error_request_count import ErrorRequestCountMetric +from aiperf.metrics.types.max_response_metric import MaxResponseTimestampMetric +from aiperf.metrics.types.min_request_metric import MinRequestTimestampMetric from aiperf.metrics.types.request_count_metric import RequestCountMetric from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric from aiperf.metrics.types.request_throughput_metric import RequestThroughputMetric from aiperf.post_processors.metric_record_processor import MetricRecordProcessor -from aiperf.post_processors.metric_results_processor import MetricResultsProcessor from tests.unit.post_processors.conftest import ( create_metric_records_message, - create_results_processor_with_metrics, setup_mock_registry_sequences, ) -TEST_LATENCY_VALUES = [100.0, 150.0, 200.0] +TEST_LATENCY_VALUES_NS = [100_000_000.0, 150_000_000.0, 200_000_000.0] TEST_REQUEST_COUNT = 100 TEST_DURATION_SECONDS = 10 EXPECTED_THROUGHPUT = TEST_REQUEST_COUNT / TEST_DURATION_SECONDS @@ -30,58 +30,45 @@ @pytest.mark.asyncio class TestPostProcessorIntegration: - """Integration tests focusing on key processor handoffs.""" + """Integration tests focusing on key metric handoffs.""" - async def test_record_to_results_data_flow( - self, - mock_metric_registry: Mock, - mock_run, - ) -> None: - """Test data flows correctly from record processor to results processor.""" - results_processor = create_results_processor_with_metrics( - mock_run, RequestLatencyMetric, RequestCountMetric - ) + async def test_record_to_accumulator_data_flow(self, mock_run) -> None: + """MetricRecordsData flows into the accumulator summary.""" + accumulator = MetricsAccumulator(mock_run) message = create_metric_records_message( x_request_id="test-1", - results=[{RequestLatencyMetric.tag: 100.0, RequestCountMetric.tag: 1}], + results=[ + {RequestLatencyMetric.tag: 100_000_000.0, RequestCountMetric.tag: 1} + ], ) - await results_processor.process_result(message.to_data()) + await accumulator.process_record(message.to_data()) + summary = await accumulator.summarize() - assert RequestLatencyMetric.tag in results_processor._results - assert isinstance( - results_processor._results[RequestLatencyMetric.tag], MetricArray - ) - assert list(results_processor._results[RequestLatencyMetric.tag].data) == [ - 100.0 - ] + assert summary.results[RequestLatencyMetric.tag].avg == pytest.approx(100.0) + assert summary.results[RequestLatencyMetric.tag].count == 1 + assert summary.results[RequestCountMetric.tag].avg == 1 - assert results_processor._results[RequestCountMetric.tag] == 1 + async def test_multiple_batches_accumulation(self, mock_run) -> None: + """The accumulator summarizes values across multiple record batches.""" + accumulator = MetricsAccumulator(mock_run) - async def test_multiple_batches_accumulation( - self, - mock_metric_registry: Mock, - mock_run, - ) -> None: - """Test accumulation across multiple record batches.""" - results_processor = create_results_processor_with_metrics( - mock_run, RequestLatencyMetric - ) - - for idx, value in enumerate(TEST_LATENCY_VALUES): + for idx, value in enumerate(TEST_LATENCY_VALUES_NS): message = create_metric_records_message( x_request_id=f"test-{idx}", request_start_ns=1_000_000_000 + idx, x_correlation_id=f"test-correlation-{idx}", results=[{RequestLatencyMetric.tag: value}], ) - await results_processor.process_result(message.to_data()) + await accumulator.process_record(message.to_data()) - assert RequestLatencyMetric.tag in results_processor._results - accumulated_data = list( - results_processor._results[RequestLatencyMetric.tag].data - ) - assert accumulated_data == TEST_LATENCY_VALUES + summary = await accumulator.summarize() + latency = summary.results[RequestLatencyMetric.tag] + + assert latency.count == len(TEST_LATENCY_VALUES_NS) + assert latency.avg == pytest.approx(150.0) + assert latency.min == pytest.approx(100.0) + assert latency.max == pytest.approx(200.0) async def test_error_metrics_isolation( self, @@ -89,7 +76,7 @@ async def test_error_metrics_isolation( mock_run, error_parsed_record: ParsedResponseRecord, ) -> None: - """Test that error and valid metrics are processed separately.""" + """Error and valid metrics are parsed by separate record-processor paths.""" setup_mock_registry_sequences( mock_metric_registry, [], [ErrorRequestCountMetric] ) @@ -106,47 +93,47 @@ async def test_error_metrics_isolation( assert ErrorRequestCountMetric.tag in result assert result[ErrorRequestCountMetric.tag] == 1 - async def test_derived_metrics_computation( - self, - mock_metric_registry: Mock, - mock_run, - ) -> None: - """Test derived metrics are computed from accumulated results.""" - setup_mock_registry_sequences( - mock_metric_registry, [RequestThroughputMetric], [] - ) - - results_processor = MetricResultsProcessor(mock_run) - - results_processor._results[RequestCountMetric.tag] = TEST_REQUEST_COUNT - results_processor._results[BenchmarkDurationMetric.tag] = ( - TEST_DURATION_SECONDS * NANOS_PER_SECOND + async def test_derived_metrics_computation(self, mock_run) -> None: + """Derived metrics are computed from accumulated aggregate metrics.""" + accumulator = MetricsAccumulator(mock_run) + await accumulator.process_record( + create_metric_records_message( + x_request_id="test-1", + results=[ + { + RequestCountMetric.tag: TEST_REQUEST_COUNT, + MinRequestTimestampMetric.tag: 0, + MaxResponseTimestampMetric.tag: ( + TEST_DURATION_SECONDS * NANOS_PER_SECOND + ), + } + ], + ).to_data() ) - await results_processor.update_derived_metrics() + summary = await accumulator.summarize() - assert RequestThroughputMetric.tag in results_processor._results - assert ( - results_processor._results[RequestThroughputMetric.tag] - == EXPECTED_THROUGHPUT + assert RequestThroughputMetric.tag in summary.results + assert summary.results[RequestThroughputMetric.tag].avg == pytest.approx( + EXPECTED_THROUGHPUT ) - async def test_complete_pipeline_summary( - self, - mock_metric_registry: Mock, - mock_run, - ) -> None: - """Test complete pipeline produces proper summary results.""" - results_processor = create_results_processor_with_metrics( - mock_run, RequestLatencyMetric - ) - - results_processor._results[RequestLatencyMetric.tag] = MetricArray() - results_processor._results[RequestLatencyMetric.tag].extend(TEST_LATENCY_VALUES) + async def test_complete_pipeline_summary(self, mock_run) -> None: + """The accumulator produces typed summary results.""" + accumulator = MetricsAccumulator(mock_run) + + for idx, value in enumerate(TEST_LATENCY_VALUES_NS): + await accumulator.process_record( + create_metric_records_message( + x_request_id=f"test-{idx}", + request_start_ns=1_000_000_000 + idx, + results=[{RequestLatencyMetric.tag: value}], + ).to_data() + ) - summary = await results_processor.summarize() + summary = await accumulator.summarize() - assert isinstance(summary, list) - assert all(hasattr(result, "tag") for result in summary) - assert all(hasattr(result, "avg") for result in summary) - assert all(hasattr(result, "count") for result in summary) + assert isinstance(summary, AccumulatorMetricsSummary) + assert all(hasattr(result, "tag") for result in summary.results.values()) + assert all(hasattr(result, "avg") for result in summary.results.values()) + assert all(hasattr(result, "count") for result in summary.results.values()) diff --git a/tests/unit/post_processors/test_record_export_jsonl_writer.py b/tests/unit/post_processors/test_record_export_jsonl_writer.py index f905510012..9e6c8c3ee0 100644 --- a/tests/unit/post_processors/test_record_export_jsonl_writer.py +++ b/tests/unit/post_processors/test_record_export_jsonl_writer.py @@ -1,25 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Smoke tests for ``RecordExportJSONLWriter``. - -Detailed behavioral coverage (initialization, process_record, file format, -HTTP trace, lifecycle) is pending. -""" +"""Smoke tests for ``RecordExportJSONLWriter``.""" from aiperf.post_processors.record_export_jsonl_writer import RecordExportJSONLWriter def test_record_export_jsonl_writer_class_importable() -> None: - """The renamed class is importable under its new path.""" + """The stream-exporter class is importable under its current path.""" assert RecordExportJSONLWriter is not None -def test_record_export_jsonl_writer_dual_dispatch_alias() -> None: - """``process_result`` aliases ``process_record`` so the writer can be - dispatched as either a legacy ``results_processor`` or a - ``stream_exporter``. - """ - assert ( - RecordExportJSONLWriter.process_result is RecordExportJSONLWriter.process_record - ) +def test_record_export_jsonl_writer_uses_stream_exporter_entrypoint() -> None: + assert callable(RecordExportJSONLWriter.process_record) + assert not hasattr(RecordExportJSONLWriter, "process_result") diff --git a/tests/unit/post_processors/test_record_export_results_processor.py b/tests/unit/post_processors/test_record_export_jsonl_writer_detailed.py similarity index 87% rename from tests/unit/post_processors/test_record_export_results_processor.py rename to tests/unit/post_processors/test_record_export_jsonl_writer_detailed.py index 2a86e93d04..0eeebdf24c 100644 --- a/tests/unit/post_processors/test_record_export_results_processor.py +++ b/tests/unit/post_processors/test_record_export_jsonl_writer_detailed.py @@ -21,8 +21,8 @@ from aiperf.config.flags.cli_config import CLIConfig from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.plugin.enums import EndpointType -from aiperf.post_processors.record_export_results_processor import ( - RecordExportResultsProcessor, +from aiperf.post_processors.record_export_jsonl_writer import ( + RecordExportJSONLWriter, ) from tests.unit.post_processors.conftest import ( aiperf_lifecycle, @@ -91,8 +91,8 @@ def sample_metric_records_message(): ) -class TestRecordExportResultsProcessorInitialization: - """Test RecordExportResultsProcessor initialization.""" +class TestRecordExportJSONLWriterInitialization: + """Test RecordExportJSONLWriter initialization.""" @pytest.mark.parametrize( "export_level, raise_exception", @@ -113,12 +113,12 @@ def test_init_with_export_level( run = make_run_with_export_level(tmp_artifact_dir, export_level) if raise_exception: with pytest.raises(PostProcessorDisabled): - _ = RecordExportResultsProcessor( + _ = RecordExportJSONLWriter( service_id="records-manager", run=run, ) else: - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run, ) @@ -132,7 +132,7 @@ def test_init_with_raw_export_level( run_records_export, ): """Test initialization with RAW export level enables the processor.""" - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -146,7 +146,7 @@ def test_init_creates_output_directory( run_records_export, ): """Test that initialization creates the output directory.""" - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -164,7 +164,7 @@ def test_init_clears_existing_file( output_file.parent.mkdir(parents=True, exist_ok=True) output_file.write_text("existing content\n") - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -186,7 +186,7 @@ def test_init_sets_show_internal_in_dev_mode( patch.object(Environment.DEV, "SHOW_INTERNAL_METRICS", True), patch.object(Environment.DEV, "SHOW_EXPERIMENTAL_METRICS", False), ): - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -194,23 +194,23 @@ def test_init_sets_show_internal_in_dev_mode( assert processor.show_internal is True -class TestRecordExportResultsProcessorProcessResult: - """Test RecordExportResultsProcessor process_result method.""" +class TestRecordExportJSONLWriterProcessResult: + """Test RecordExportJSONLWriter process_record method.""" @pytest.mark.asyncio - async def test_process_result_writes_valid_data( + async def test_process_record_writes_valid_data( self, run_records_export, sample_metric_records_message: MetricRecordsMessage, mock_metric_registry: Mock, ): - """Test that process_result writes valid data to file.""" + """Test that process_record writes valid data to file.""" mock_display_dict = { "request_latency": MetricValue(value=1.0, unit="ms"), "output_token_count": MetricValue(value=10, unit="tokens"), } - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -221,7 +221,7 @@ async def test_process_result_writes_valid_data( "to_display_dict", return_value=mock_display_dict, ): - await processor.process_result(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message.to_data()) lines = processor.output_file.read_text().splitlines() @@ -240,21 +240,21 @@ async def test_process_result_writes_valid_data( assert "output_token_count" in record.metrics @pytest.mark.asyncio - async def test_process_result_with_empty_display_metrics( + async def test_process_record_with_empty_display_metrics( self, run_records_export, sample_metric_records_message: MetricRecordsMessage, mock_metric_registry: Mock, ): - """Test that process_result skips records with empty display metrics.""" - processor = RecordExportResultsProcessor( + """Test that process_record skips records with empty display metrics.""" + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) # Mock to_display_dict to return empty dict with patch.object(MetricRecordDict, "to_display_dict", return_value={}): - await processor.process_result(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message.to_data()) # Should not write anything since display_metrics is empty assert processor.lines_written == 0 @@ -263,14 +263,14 @@ async def test_process_result_with_empty_display_metrics( assert content == "" @pytest.mark.asyncio - async def test_process_result_handles_errors_gracefully( + async def test_process_record_handles_errors_gracefully( self, run_records_export, sample_metric_records_message: MetricRecordsMessage, mock_metric_registry: Mock, ): """Test that errors during processing don't raise exceptions.""" - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -283,7 +283,7 @@ async def test_process_result_handles_errors_gracefully( patch.object(processor, "error") as mock_error, ): # Should not raise - await processor.process_result(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message.to_data()) # Should log the error assert mock_error.call_count >= 1 @@ -292,7 +292,7 @@ async def test_process_result_handles_errors_gracefully( assert processor.lines_written == 0 @pytest.mark.asyncio - async def test_process_result_multiple_messages( + async def test_process_record_multiple_messages( self, run_records_export, sample_metric_records_message: MetricRecordsMessage, @@ -303,7 +303,7 @@ async def test_process_result_multiple_messages( "request_latency": MetricValue(value=1.0, unit="ms"), } - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -320,7 +320,7 @@ async def test_process_result_multiple_messages( request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}, {"metric2": 200}], ) - await processor.process_result(message.to_data()) + await processor.process_record(message.to_data()) assert processor.lines_written == 5 assert processor.output_file.exists() @@ -337,8 +337,8 @@ async def test_process_result_multiple_messages( assert "request_latency" in record.metrics -class TestRecordExportResultsProcessorFileFormat: - """Test RecordExportResultsProcessor file format.""" +class TestRecordExportJSONLWriterFileFormat: + """Test RecordExportJSONLWriter file format.""" @pytest.mark.asyncio async def test_output_is_valid_jsonl( @@ -350,7 +350,7 @@ async def test_output_is_valid_jsonl( """Test that output file is valid JSONL format.""" mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -359,7 +359,7 @@ async def test_output_is_valid_jsonl( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_result(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message.to_data()) lines = processor.output_file.read_text().splitlines() @@ -380,7 +380,7 @@ async def test_record_structure_is_complete( """Test that each record has the expected structure.""" mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -389,7 +389,7 @@ async def test_record_structure_is_complete( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_result(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message.to_data()) lines = processor.output_file.read_text().splitlines() @@ -413,8 +413,8 @@ async def test_record_structure_is_complete( assert record.metrics["test_metric"].unit == "ms" -class TestRecordExportResultsProcessorLogging: - """Test RecordExportResultsProcessor logging behavior.""" +class TestRecordExportJSONLWriterLogging: + """Test RecordExportJSONLWriter logging behavior.""" @pytest.mark.asyncio async def test_periodic_debug_logging( @@ -426,7 +426,7 @@ async def test_periodic_debug_logging( """Test that debug logging occurs when buffer is flushed.""" mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -444,7 +444,7 @@ async def test_periodic_debug_logging( request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}, {"metric2": 200}], ) - await processor.process_result(message.to_data()) + await processor.process_record(message.to_data()) # Wait for async flush task to complete await processor.wait_for_tasks() @@ -460,7 +460,7 @@ async def test_error_logging_on_write_failure( mock_metric_registry: Mock, ): """Test that errors are logged when write fails.""" - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -471,15 +471,15 @@ async def test_error_logging_on_write_failure( ), patch.object(processor, "error") as mock_error, ): - await processor.process_result(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message.to_data()) assert mock_error.call_count >= 1 call_args = str(mock_error.call_args_list[0]) assert "Failed to write record metrics" in call_args -class TestRecordExportResultsProcessorShutdown: - """Test RecordExportResultsProcessor shutdown behavior.""" +class TestRecordExportJSONLWriterShutdown: + """Test RecordExportJSONLWriter shutdown behavior.""" @pytest.mark.asyncio async def test_shutdown_logs_statistics( @@ -491,7 +491,7 @@ async def test_shutdown_logs_statistics( """Test that shutdown logs final statistics.""" mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -511,7 +511,7 @@ async def test_shutdown_logs_statistics( request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}], ) - await processor.process_result(message.to_data()) + await processor.process_record(message.to_data()) # Wait for any pending flush tasks await processor.wait_for_tasks() @@ -527,8 +527,8 @@ async def test_shutdown_logs_statistics( raise -class TestRecordExportResultsProcessorSummarize: - """Test RecordExportResultsProcessor summarize method.""" +class TestRecordExportJSONLWriterSummarize: + """Test RecordExportJSONLWriter summarize method.""" @pytest.mark.asyncio async def test_summarize_returns_empty_list( @@ -536,7 +536,7 @@ async def test_summarize_returns_empty_list( run_records_export, ): """Test that summarize returns an empty list (no aggregation needed).""" - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -547,8 +547,8 @@ async def test_summarize_returns_empty_list( assert isinstance(result, list) -class TestRecordExportResultsProcessorHttpTrace: - """Test RecordExportResultsProcessor HTTP trace export functionality.""" +class TestRecordExportJSONLWriterHttpTrace: + """Test RecordExportJSONLWriter HTTP trace export functionality.""" @pytest.fixture def cfg_with_http_trace(self, tmp_artifact_dir: Path) -> CLIConfig: @@ -617,7 +617,7 @@ def test_init_default_http_trace_disabled( run_records_export, ): """Test that export_http_trace defaults to False.""" - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -629,7 +629,7 @@ def test_init_http_trace_enabled( run_with_http_trace, ): """Test that export_http_trace can be enabled via config.""" - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_with_http_trace, ) @@ -643,7 +643,7 @@ def test_init_logs_when_http_trace_enabled( ): """Test that initialization logs when HTTP trace export is enabled.""" with caplog.at_level(logging.INFO): - _ = RecordExportResultsProcessor( + _ = RecordExportJSONLWriter( service_id="records-manager", run=run_with_http_trace, ) @@ -660,7 +660,7 @@ async def test_trace_data_excluded_when_disabled( """Test that trace_data is NOT in output when export_http_trace=False.""" mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -677,7 +677,7 @@ async def test_trace_data_excluded_when_disabled( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_result(message.to_data()) + await processor.process_record(message.to_data()) lines = processor.output_file.read_text().splitlines() assert len(lines) == 1 @@ -700,7 +700,7 @@ async def test_trace_data_included_when_enabled( """Test that trace_data IS included in output when export_http_trace=True.""" mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_with_http_trace, ) @@ -717,7 +717,7 @@ async def test_trace_data_included_when_enabled( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_result(message.to_data()) + await processor.process_record(message.to_data()) lines = processor.output_file.read_text().splitlines() assert len(lines) == 1 @@ -748,13 +748,13 @@ async def test_metrics_always_present_regardless_of_trace_flag( } # Test with trace disabled - processor_disabled = RecordExportResultsProcessor( + processor_disabled = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) # Test with trace enabled - processor_enabled = RecordExportResultsProcessor( + processor_enabled = RecordExportJSONLWriter( service_id="records-manager", run=run_with_http_trace, ) @@ -771,7 +771,7 @@ async def test_metrics_always_present_regardless_of_trace_flag( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_result(message.to_data()) + await processor.process_record(message.to_data()) lines = processor.output_file.read_text().splitlines() assert len(lines) == 1 @@ -794,7 +794,7 @@ async def test_no_trace_data_when_record_has_none( """Test trace_data is null when record has no trace data (even if enabled).""" mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_with_http_trace, ) @@ -811,7 +811,7 @@ async def test_no_trace_data_when_record_has_none( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_result(message.to_data()) + await processor.process_record(message.to_data()) lines = processor.output_file.read_text().splitlines() assert len(lines) == 1 @@ -823,8 +823,8 @@ async def test_no_trace_data_when_record_has_none( assert record.trace_data is None -class TestRecordExportResultsProcessorLifecycle: - """Test RecordExportResultsProcessor lifecycle.""" +class TestRecordExportJSONLWriterLifecycle: + """Test RecordExportJSONLWriter lifecycle.""" @pytest.mark.asyncio async def test_lifecycle( @@ -834,7 +834,7 @@ async def test_lifecycle( mock_aiofiles_stringio, ): """Test that the processor can be initialized, processed, and shutdown.""" - processor = RecordExportResultsProcessor( + processor = RecordExportJSONLWriter( service_id="records-manager", run=run_records_export, ) @@ -851,7 +851,7 @@ async def test_lifecycle( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): for i in range(Environment.RECORD.EXPORT_BATCH_SIZE * 2): - await processor.process_result( + await processor.process_record( create_metric_records_message( x_request_id=f"record-{i}", conversation_id=f"conv-{i}", @@ -884,12 +884,7 @@ async def test_lifecycle( class TestRecordExportSingleRegistration: - """Double-write reversion guard: the per-record JSONL must be produced by - exactly ONE registered plugin. RecordExportResultsProcessor (results_processor) - and RecordExportJSONLWriter (stream_exporter) once BOTH wrote it, duplicating - every record. The results_processor registration was removed; the class still - exists for direct use/tests but must NOT be auto-instantiated by the registry. - """ + """Double-write reversion guard for per-record JSONL export.""" def test_record_export_registered_exactly_once(self) -> None: from aiperf.plugin import plugins @@ -904,9 +899,8 @@ def test_record_export_registered_exactly_once(self) -> None: f"reintroduces the per-record double-write): {record_export_entries}" ) - def test_results_processor_has_no_record_export(self) -> None: + def test_legacy_results_processor_category_is_absent(self) -> None: from aiperf.plugin import plugins - from aiperf.plugin.enums import PluginType - names = [e.name for e in plugins.iter_entries(PluginType.RESULTS_PROCESSOR)] - assert "record_export" not in names + categories = {entry.category for entry in plugins.iter_entries()} + assert "results_processor" not in categories diff --git a/tests/unit/post_processors/test_timeslice_metric_results_processor.py b/tests/unit/post_processors/test_timeslice_metric_results_processor.py index 7d463ef4c5..17cdc7d6ef 100644 --- a/tests/unit/post_processors/test_timeslice_metric_results_processor.py +++ b/tests/unit/post_processors/test_timeslice_metric_results_processor.py @@ -1,374 +1,227 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from unittest.mock import Mock, patch +from __future__ import annotations import pytest from aiperf.common.constants import NANOS_PER_SECOND -from aiperf.common.enums import MetricType -from aiperf.common.exceptions import NoMetricValue, PostProcessorDisabled -from aiperf.common.models import MetricResult -from aiperf.metrics.metric_dicts import MetricArray, MetricResultsDict +from aiperf.metrics.accumulator import MetricsAccumulator +from aiperf.metrics.types.max_response_metric import MaxResponseTimestampMetric +from aiperf.metrics.types.min_request_metric import MinRequestTimestampMetric from aiperf.metrics.types.request_count_metric import RequestCountMetric from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric from aiperf.metrics.types.request_throughput_metric import RequestThroughputMetric -from aiperf.post_processors.timeslice_metric_results_processor import ( - TimesliceMetricResultsProcessor, -) from tests.unit.post_processors.conftest import create_metric_records_message -class TestTimesliceMetricResultsProcessor: - """Test cases for TimesliceMetricResultsProcessor.""" +class TestMetricsAccumulatorTimeslices: + """Timeslice coverage for the accumulator-backed summary path.""" - def test_initialization_without_slice_duration_raises_exception( - self, mock_metric_registry: Mock, mock_run + def test_initialization_without_slice_duration_disables_timeslices( + self, mock_run ) -> None: - """Test that processor initialization fails when slice_duration is not set.""" - # Ensure slice_duration is None mock_run.cfg.artifacts.slice_duration = None + accumulator = MetricsAccumulator(mock_run) - with pytest.raises(PostProcessorDisabled, match="requires slice_duration"): - TimesliceMetricResultsProcessor(mock_run) + assert accumulator._slice_duration_ns is None - def test_initialization_with_slice_duration( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test processor initialization sets up timeslice-specific data structures.""" - mock_run.cfg.artifacts.slice_duration = 1.0 - processor = TimesliceMetricResultsProcessor(mock_run) - - assert hasattr(processor, "_timeslice_instances_maps") - assert hasattr(processor, "_timeslice_results") - assert hasattr(processor, "_slice_duration_ns") - assert processor._slice_duration_ns == 1.0 * NANOS_PER_SECOND - - @pytest.mark.asyncio - async def test_get_instances_map_requires_request_start_ns( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test that get_instances_map raises ValueError when request_start_ns is None.""" + def test_initialization_with_slice_duration(self, mock_run) -> None: mock_run.cfg.artifacts.slice_duration = 1.0 - processor = TimesliceMetricResultsProcessor(mock_run) + accumulator = MetricsAccumulator(mock_run) - with pytest.raises(ValueError, match="must be passed a request_start_ns"): - await processor.get_instances_map(None) + assert accumulator._slice_duration_ns == NANOS_PER_SECOND @pytest.mark.asyncio - async def test_get_results_requires_request_start_ns( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test that get_results raises ValueError when request_start_ns is None.""" + async def test_process_record_separates_by_timeslice(self, mock_run) -> None: mock_run.cfg.artifacts.slice_duration = 1.0 - processor = TimesliceMetricResultsProcessor(mock_run) - - with pytest.raises(ValueError, match="must be passed a request_start_ns"): - await processor.get_results(None) - - @pytest.mark.asyncio - async def test_process_result_separates_by_timeslice( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test that metrics are separated into different timeslices based on timestamp.""" - mock_run.cfg.artifacts.slice_duration = 1.0 # 1 second - processor = TimesliceMetricResultsProcessor(mock_run) - processor._tags_to_types = {"test_record": MetricType.RECORD} - - # Process request in first timeslice (0.5 seconds) - message1 = create_metric_records_message( - x_request_id="test-1", - request_start_ns=int(0.5 * NANOS_PER_SECOND), - results=[{"test_record": 42.0}], + accumulator = MetricsAccumulator(mock_run) + + await accumulator.process_record( + create_metric_records_message( + x_request_id="test-1", + request_start_ns=int(0.5 * NANOS_PER_SECOND), + request_end_ns=int(0.6 * NANOS_PER_SECOND), + results=[{RequestLatencyMetric.tag: 42_000_000.0}], + ).to_data() ) - await processor.process_result(message1.to_data()) - - # Process request in second timeslice (1.5 seconds) - message2 = create_metric_records_message( - x_request_id="test-2", - request_start_ns=int(1.5 * NANOS_PER_SECOND), - results=[{"test_record": 84.0}], + await accumulator.process_record( + create_metric_records_message( + x_request_id="test-2", + request_start_ns=int(1.5 * NANOS_PER_SECOND), + request_end_ns=int(1.6 * NANOS_PER_SECOND), + results=[{RequestLatencyMetric.tag: 84_000_000.0}], + ).to_data() ) - await processor.process_result(message2.to_data()) - # Verify results are in different timeslices - assert 0 in processor._timeslice_results - assert 1 in processor._timeslice_results - assert list(processor._timeslice_results[0]["test_record"].data) == [42.0] - assert list(processor._timeslice_results[1]["test_record"].data) == [84.0] + timeslices = (await accumulator.summarize()).timeslices - @pytest.mark.asyncio - async def test_process_result_accumulates_in_same_timeslice( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test that metrics in the same timeslice are accumulated together.""" - mock_run.cfg.artifacts.slice_duration = 1.0 # 1 second - processor = TimesliceMetricResultsProcessor(mock_run) - processor._tags_to_types = {"test_record": MetricType.RECORD} - - # Process two requests in same timeslice (both in first second) - message1 = create_metric_records_message( - x_request_id="test-1", - request_start_ns=int(0.3 * NANOS_PER_SECOND), - results=[{"test_record": 10.0}], - ) - await processor.process_result(message1.to_data()) - - message2 = create_metric_records_message( - x_request_id="test-2", - request_start_ns=int(0.7 * NANOS_PER_SECOND), - results=[{"test_record": 20.0}], - ) - await processor.process_result(message2.to_data()) - - # Verify results are accumulated in same timeslice - assert 0 in processor._timeslice_results - assert list(processor._timeslice_results[0]["test_record"].data) == [10.0, 20.0] - - @pytest.mark.asyncio - async def test_process_result_aggregate_metric_per_timeslice( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test that aggregate metrics work correctly per timeslice.""" - mock_run.cfg.artifacts.slice_duration = 1.0 # 1 second - processor = TimesliceMetricResultsProcessor(mock_run) - processor._tags_to_types = {RequestCountMetric.tag: MetricType.AGGREGATE} - - # First timeslice - two requests - message1 = create_metric_records_message( - x_request_id="test-1", - request_start_ns=int(0.5 * NANOS_PER_SECOND), - results=[{RequestCountMetric.tag: 5}], - ) - await processor.process_result(message1.to_data()) - - message2 = create_metric_records_message( - x_request_id="test-2", - request_start_ns=int(0.7 * NANOS_PER_SECOND), - results=[{RequestCountMetric.tag: 3}], - ) - await processor.process_result(message2.to_data()) - - # Second timeslice - one request - message3 = create_metric_records_message( - x_request_id="test-3", - request_start_ns=int(1.5 * NANOS_PER_SECOND), - results=[{RequestCountMetric.tag: 7}], - ) - await processor.process_result(message3.to_data()) - - # Verify aggregate counts are separate per timeslice - assert processor._timeslice_results[0][RequestCountMetric.tag] == 8 - assert processor._timeslice_results[1][RequestCountMetric.tag] == 7 + assert timeslices is not None + assert len(timeslices) == 2 + assert timeslices[0].metric_results[RequestLatencyMetric.tag].avg == 42.0 + assert timeslices[1].metric_results[RequestLatencyMetric.tag].avg == 84.0 @pytest.mark.asyncio - async def test_timeslice_boundary_conditions( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test behavior at timeslice boundaries.""" - mock_run.cfg.artifacts.slice_duration = 1.0 # 1 second - processor = TimesliceMetricResultsProcessor(mock_run) - processor._tags_to_types = {"test_record": MetricType.RECORD} - - # Request at 0.999s (should be in timeslice 0) - message1 = create_metric_records_message( - x_request_id="test-1", - request_start_ns=int(0.999 * NANOS_PER_SECOND), - results=[{"test_record": 1.0}], - ) - await processor.process_result(message1.to_data()) + async def test_process_record_accumulates_in_same_timeslice(self, mock_run) -> None: + mock_run.cfg.artifacts.slice_duration = 1.0 + accumulator = MetricsAccumulator(mock_run) + + for idx, value in enumerate([10_000_000.0, 20_000_000.0]): + await accumulator.process_record( + create_metric_records_message( + x_request_id=f"test-{idx}", + request_start_ns=int((0.3 + idx * 0.4) * NANOS_PER_SECOND), + request_end_ns=int((0.35 + idx * 0.4) * NANOS_PER_SECOND), + results=[{RequestLatencyMetric.tag: value}], + ).to_data() + ) - # Request at 1.0s (should be in timeslice 1) - message2 = create_metric_records_message( - x_request_id="test-2", - request_start_ns=int(1.0 * NANOS_PER_SECOND), - results=[{"test_record": 2.0}], - ) - await processor.process_result(message2.to_data()) + timeslices = (await accumulator.summarize()).timeslices - # Request at 1.001s (should be in timeslice 1) - message3 = create_metric_records_message( - x_request_id="test-3", - request_start_ns=int(1.001 * NANOS_PER_SECOND), - results=[{"test_record": 3.0}], - ) - await processor.process_result(message3.to_data()) - - # Verify proper separation at boundaries - assert list(processor._timeslice_results[0]["test_record"].data) == [1.0] - assert list(processor._timeslice_results[1]["test_record"].data) == [2.0, 3.0] + assert timeslices is not None + assert len(timeslices) == 1 + result = timeslices[0].metric_results[RequestLatencyMetric.tag] + assert result.count == 2 + assert result.avg == pytest.approx(15.0) @pytest.mark.asyncio - async def test_update_derived_metrics_per_timeslice( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test that derived metrics are computed per timeslice.""" - - def mock_derive_func(results_dict: MetricResultsDict): - # Simple derive func that returns a constant based on existence of data - return 100.0 if results_dict else 0.0 - + async def test_aggregate_metric_per_timeslice(self, mock_run) -> None: mock_run.cfg.artifacts.slice_duration = 1.0 - processor = TimesliceMetricResultsProcessor(mock_run) - processor.derive_funcs = {RequestThroughputMetric.tag: mock_derive_func} - - # Set up some dummy results in different timeslices - processor._timeslice_results[0]["base_metric"] = 42 - processor._timeslice_results[1]["base_metric"] = 84 + accumulator = MetricsAccumulator(mock_run) + + records = [ + (0.5, 5), + (0.7, 3), + (1.5, 7), + ] + for idx, (start_s, count) in enumerate(records): + await accumulator.process_record( + create_metric_records_message( + x_request_id=f"test-{idx}", + request_start_ns=int(start_s * NANOS_PER_SECOND), + request_end_ns=int((start_s + 0.1) * NANOS_PER_SECOND), + results=[{RequestCountMetric.tag: count}], + ).to_data() + ) - await processor.update_derived_metrics() + timeslices = (await accumulator.summarize()).timeslices - # Verify derived metrics are computed for each timeslice - assert processor._timeslice_results[0][RequestThroughputMetric.tag] == 100.0 - assert processor._timeslice_results[1][RequestThroughputMetric.tag] == 100.0 + assert timeslices is not None + assert timeslices[0].metric_results[RequestCountMetric.tag].avg == 8 + assert timeslices[1].metric_results[RequestCountMetric.tag].avg == 7 @pytest.mark.asyncio - async def test_update_derived_metrics_handles_no_metric_value( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test that NoMetricValue exceptions are caught and logged gracefully per timeslice.""" - - def failing_derive_func(results_dict: MetricResultsDict): - raise NoMetricValue("Cannot derive value") - + async def test_timeslice_boundary_conditions(self, mock_run) -> None: mock_run.cfg.artifacts.slice_duration = 1.0 - processor = TimesliceMetricResultsProcessor(mock_run) - processor.derive_funcs = {RequestThroughputMetric.tag: failing_derive_func} - processor._timeslice_results[0]["base_metric"] = 42 + accumulator = MetricsAccumulator(mock_run) + + records = [ + (0.999, 1_000_000.0), + (1.0, 2_000_000.0), + (1.001, 3_000_000.0), + ] + for idx, (start_s, value) in enumerate(records): + await accumulator.process_record( + create_metric_records_message( + x_request_id=f"test-{idx}", + request_start_ns=int(start_s * NANOS_PER_SECOND), + request_end_ns=int((start_s + 0.01) * NANOS_PER_SECOND), + results=[{RequestLatencyMetric.tag: value}], + ).to_data() + ) - with patch.object(processor, "debug") as mock_debug: - # NoMetricValue should be caught and logged, not raised - await processor.update_derived_metrics() + timeslices = (await accumulator.summarize()).timeslices - # Verify no derived metric was added (exception was caught) - assert RequestThroughputMetric.tag not in processor._timeslice_results[0] - # Verify the exception was logged via debug - mock_debug.assert_called_once() - assert "No metric value" in str(mock_debug.call_args) + assert timeslices is not None + assert len(timeslices) == 1 + result = timeslices[0].metric_results[RequestLatencyMetric.tag] + assert result.count == 3 + assert result.avg == pytest.approx(2.0) @pytest.mark.asyncio - async def test_update_derived_metrics_handles_value_error( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test that derived metrics handle ValueError exceptions gracefully.""" - - def failing_derive_func(results_dict: MetricResultsDict): - raise ValueError("Calculation error") - + async def test_derived_metrics_are_computed_per_timeslice(self, mock_run) -> None: mock_run.cfg.artifacts.slice_duration = 1.0 - processor = TimesliceMetricResultsProcessor(mock_run) - processor.derive_funcs = {RequestThroughputMetric.tag: failing_derive_func} - processor._timeslice_results[0]["base_metric"] = 42 - - with patch.object(processor, "warning") as mock_warning: - await processor.update_derived_metrics() + accumulator = MetricsAccumulator(mock_run) + + records = [ + (0.5, 1.5, 5), + (1.5, 2.5, 10), + ] + for idx, (start_s, end_s, count) in enumerate(records): + await accumulator.process_record( + create_metric_records_message( + x_request_id=f"test-{idx}", + request_start_ns=int(start_s * NANOS_PER_SECOND), + request_end_ns=int(end_s * NANOS_PER_SECOND), + results=[ + { + RequestCountMetric.tag: count, + MinRequestTimestampMetric.tag: start_s * NANOS_PER_SECOND, + MaxResponseTimestampMetric.tag: end_s * NANOS_PER_SECOND, + } + ], + ).to_data() + ) - # Verify no derived metric was added - assert RequestThroughputMetric.tag not in processor._timeslice_results[0] - mock_warning.assert_called() + timeslices = (await accumulator.summarize()).timeslices - @pytest.mark.asyncio - async def test_summarize_returns_dict_of_timeslices( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test summarize returns dict mapping timeslice indices to metric results.""" - mock_run.cfg.artifacts.slice_duration = 1.0 - processor = TimesliceMetricResultsProcessor(mock_run) - processor._tags_to_types = {RequestLatencyMetric.tag: MetricType.RECORD} - - # Set up results in multiple timeslices - # Values are in nanoseconds (internal unit), summarize() converts to ms (display unit) - processor._timeslice_results[0][RequestLatencyMetric.tag] = MetricArray() - processor._timeslice_results[0][RequestLatencyMetric.tag].append(42_000_000.0) - - processor._timeslice_results[1][RequestLatencyMetric.tag] = MetricArray() - processor._timeslice_results[1][RequestLatencyMetric.tag].append(84_000_000.0) - - # Set up the instances map (used by _create_metric_result) - # The parent class _create_metric_result uses self._instances_map - processor._instances_map = {RequestLatencyMetric.tag: RequestLatencyMetric()} - - results = await processor.summarize() - - # Verify structure: dict of timeslice_index -> list[MetricResult] - assert isinstance(results, dict) - assert 0 in results - assert 1 in results - assert isinstance(results[0], list) - assert isinstance(results[1], list) - assert len(results[0]) == 1 - assert len(results[1]) == 1 - assert isinstance(results[0][0], MetricResult) - assert isinstance(results[1][0], MetricResult) - assert results[0][0].tag == RequestLatencyMetric.tag - assert results[1][0].tag == RequestLatencyMetric.tag - # Verify the actual values are in display units (ms) - assert results[0][0].avg == 42.0 - assert results[1][0].avg == 84.0 - assert results[0][0].unit == "ms" - assert results[1][0].unit == "ms" + assert timeslices is not None + assert timeslices[0].metric_results[RequestThroughputMetric.tag].avg == 5.0 + assert timeslices[1].metric_results[RequestThroughputMetric.tag].avg == 10.0 @pytest.mark.asyncio - async def test_summarize_with_empty_timeslices( - self, mock_metric_registry: Mock, mock_run + async def test_summarize_without_records_returns_no_timeslices( + self, mock_run ) -> None: - """Test summarize handles empty timeslices correctly.""" mock_run.cfg.artifacts.slice_duration = 1.0 - processor = TimesliceMetricResultsProcessor(mock_run) + accumulator = MetricsAccumulator(mock_run) - # No data processed - results = await processor.summarize() + summary = await accumulator.summarize() - # Should return empty dict - assert isinstance(results, dict) - assert len(results) == 0 + assert summary.timeslices is None @pytest.mark.asyncio async def test_multiple_timeslices_with_different_slice_duration( - self, mock_metric_registry: Mock, mock_run + self, mock_run ) -> None: - """Test that a different slice_duration value works correctly.""" - # Test with 500ms slices (different from default 1000ms) mock_run.cfg.artifacts.slice_duration = 0.5 - processor = TimesliceMetricResultsProcessor(mock_run) - processor._tags_to_types = {"test_record": MetricType.RECORD} + accumulator = MetricsAccumulator(mock_run) - # Process requests across multiple 0.5s slices for i in range(4): - message = create_metric_records_message( - x_request_id=f"test-{i}", - request_start_ns=int((i * 0.5 + 0.25) * NANOS_PER_SECOND), - results=[{"test_record": float(i)}], + start_s = i * 0.5 + 0.25 + await accumulator.process_record( + create_metric_records_message( + x_request_id=f"test-{i}", + request_start_ns=int(start_s * NANOS_PER_SECOND), + request_end_ns=int((start_s + 0.05) * NANOS_PER_SECOND), + results=[{RequestLatencyMetric.tag: float(i * 1_000_000)}], + ).to_data() ) - await processor.process_result(message.to_data()) - # Should have 4 different timeslices (0, 1, 2, 3) - assert len(processor._timeslice_results) == 4 - for i in range(4): - assert i in processor._timeslice_results - assert list(processor._timeslice_results[i]["test_record"].data) == [ - float(i) - ] + timeslices = (await accumulator.summarize()).timeslices + + assert timeslices is not None + assert len(timeslices) == 4 + assert [ + ts.metric_results[RequestLatencyMetric.tag].avg for ts in timeslices + ] == [0.0, 1.0, 2.0, 3.0] @pytest.mark.asyncio - async def test_timeslice_instances_map_creates_separate_instances( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test that each timeslice gets its own metric instances.""" + async def test_trailing_partial_timeslice_is_flagged(self, mock_run) -> None: mock_run.cfg.artifacts.slice_duration = 1.0 - processor = TimesliceMetricResultsProcessor(mock_run) - processor._tags_to_types = {RequestCountMetric.tag: MetricType.AGGREGATE} - - # Get instances for two different timestamps in different timeslices - request_start_ns_1 = int(0.5 * NANOS_PER_SECOND) - request_start_ns_2 = int(1.5 * NANOS_PER_SECOND) + accumulator = MetricsAccumulator(mock_run) + + await accumulator.process_record( + create_metric_records_message( + x_request_id="test-1", + request_start_ns=int(0.5 * NANOS_PER_SECOND), + request_end_ns=int(0.6 * NANOS_PER_SECOND), + results=[{RequestLatencyMetric.tag: 42_000_000.0}], + ).to_data() + ) - instances_map_0 = await processor.get_instances_map(request_start_ns_1) - instances_map_1 = await processor.get_instances_map(request_start_ns_2) + timeslices = (await accumulator.summarize()).timeslices - # Verify they are different instances - assert instances_map_0 is not instances_map_1 - assert ( - instances_map_0[RequestCountMetric.tag] - is not instances_map_1[RequestCountMetric.tag] - ) + assert timeslices is not None + assert timeslices[0].is_complete is False + assert timeslices[0].end_ns == int(0.6 * NANOS_PER_SECOND) diff --git a/tests/unit/property/_numeric_bounds_baseline.txt b/tests/unit/property/_numeric_bounds_baseline.txt index 0b607afe32..81e4a07680 100644 --- a/tests/unit/property/_numeric_bounds_baseline.txt +++ b/tests/unit/property/_numeric_bounds_baseline.txt @@ -283,7 +283,6 @@ ProfileResults.completed: numeric field has no ge/gt/le/lt bound and is not Fini ProfileResults.end_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. ProfileResultsMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. ProfileResults.start_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. -ProfileResults.timeslice_metric_results: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. ProfileResults.total_expected: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. ProfileStartCommand.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. PromptConfig.isl: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. diff --git a/tests/unit/records/test_records_manager.py b/tests/unit/records/test_records_manager.py index 595bf3a864..4873ae1a91 100644 --- a/tests/unit/records/test_records_manager.py +++ b/tests/unit/records/test_records_manager.py @@ -1,19 +1,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import asyncio from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock import pytest from aiperf.common.enums import CreditPhase -from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.messages import BaseServiceErrorMessage from aiperf.common.messages.inference_messages import ( MetricRecordsData, MetricRecordsMessage, ) +from aiperf.common.messages.telemetry_messages import TelemetryRecordsMessage from aiperf.common.models import ( BranchStats, CreditPhaseStats, @@ -22,9 +24,10 @@ ProfileResults, TelemetryMetrics, TelemetryRecord, + TimesliceResult, ) +from aiperf.common.models.error_models import ErrorDetails from aiperf.common.models.record_models import MetricRecordMetadata -from aiperf.common.models.server_metrics_models import ServerMetricsRecord from aiperf.common.types import MetricTagT from aiperf.credit.messages import ( CreditPhaseCompleteMessage, @@ -36,28 +39,11 @@ from aiperf.metrics.accumulator_models import AccumulatorMetricsSummary from aiperf.metrics.cache_reporting_hint import CACHE_REPORTING_HINT from aiperf.plugin.enums import AccumulatorType, TimingMode -from aiperf.post_processors.metric_results_processor import ( - MetricResultsProcessor as CanonicalMetricResultsProcessor, -) from aiperf.records.records_manager import ErrorTrackingState, RecordsManager from aiperf.records.records_tracker import RecordsTracker from aiperf.timing.config import CreditPhaseConfig -from tests.harness import mock_plugin - # Helper functions -def create_mock_records_manager( - start_time_ns: int, - expected_duration_sec: float | None, - grace_period_sec: float = 0.0, -) -> MagicMock: - """Create a mock RecordsManager instance for testing filtering logic.""" - instance = MagicMock() - instance.expected_duration_sec = expected_duration_sec - instance.start_time_ns = start_time_ns - instance.cli_config.benchmark_grace_period = grace_period_sec - instance.debug = MagicMock() - return instance def create_metric_record_data( @@ -81,35 +67,26 @@ def create_metric_record_data( ) -class TestRecordsManagerTelemetry: - """Test RecordsManager telemetry handling with mocked components.""" +def _telemetry_record(gpu_index: int = 0) -> TelemetryRecord: + return TelemetryRecord( + timestamp_ns=1_000_000 + gpu_index, + dcgm_url="http://localhost:9400/metrics", + gpu_index=gpu_index, + gpu_uuid=f"GPU-{gpu_index}", + gpu_model_name="Test GPU", + telemetry_data=TelemetryMetrics(gpu_power_usage=100.0), + ) - @pytest.mark.asyncio - async def test_on_telemetry_records_valid(self): - """Test handling valid telemetry records.""" - from unittest.mock import AsyncMock, MagicMock - - from aiperf.common.messages import TelemetryRecordsMessage - from aiperf.common.models import ( - TelemetryHierarchy, - TelemetryMetrics, - TelemetryRecord, - ) - # Create sample telemetry records - records = [ - TelemetryRecord( - timestamp_ns=1000000, - dcgm_url="http://localhost:9400/metrics", - gpu_index=0, - gpu_uuid="GPU-123", - gpu_model_name="Test GPU", - telemetry_data=TelemetryMetrics( - gpu_power_usage=100.0, - ), - ) - ] +class TestRecordsManagerTelemetry: + """Telemetry records route through the unified record dispatcher.""" + @pytest.mark.asyncio + async def test_on_telemetry_records_valid_dispatches_each_record(self) -> None: + manager = RecordsManager.__new__(RecordsManager) + manager._telemetry_state = ErrorTrackingState() + manager._dispatch_record = AsyncMock(return_value=[]) + records = [_telemetry_record(0), _telemetry_record(1)] message = TelemetryRecordsMessage( service_id="test_service", collector_id="test_collector", @@ -118,129 +95,73 @@ async def test_on_telemetry_records_valid(self): error=None, ) - # Mock the hierarchy - mock_hierarchy = MagicMock(spec=TelemetryHierarchy) - mock_hierarchy.add_record = MagicMock() - mock_send_to_processors = AsyncMock() - - # Test the logic directly without instantiating the full service - for record in message.records: - mock_hierarchy.add_record(record) - - if message.records: - await mock_send_to_processors(message.records) + await manager._on_telemetry_records(message) - # Verify behavior - assert mock_hierarchy.add_record.call_count == len(records) - mock_send_to_processors.assert_called_once_with(records) + assert manager._dispatch_record.await_args_list == [ + ((records[0],),), + ((records[1],),), + ] + assert manager._telemetry_state.error_counts == {} @pytest.mark.asyncio - async def test_on_telemetry_records_invalid(self): - """Test handling invalid telemetry records with errors.""" - from unittest.mock import AsyncMock - - from aiperf.common.messages import TelemetryRecordsMessage - from aiperf.common.models import ErrorDetails - - error = ErrorDetails(message="Test error", code=500) + async def test_on_telemetry_dispatch_errors_are_tracked(self) -> None: + manager = RecordsManager.__new__(RecordsManager) + manager._telemetry_state = ErrorTrackingState() + dispatch_error = RuntimeError("telemetry writer failed") + manager._dispatch_record = AsyncMock(return_value=[dispatch_error]) - message = TelemetryRecordsMessage( - service_id="test_service", - collector_id="test_collector", - dcgm_url="http://localhost:9400/metrics", - records=[], - error=error, + await manager._on_telemetry_records( + TelemetryRecordsMessage( + service_id="test_service", + collector_id="test_collector", + dcgm_url="http://localhost:9400/metrics", + records=[_telemetry_record()], + error=None, + ) ) - mock_send_to_processors = AsyncMock() - error_counts = {} - - # Test the logic: errors should be tracked, not sent to processors - if message.error: - error_counts[message.error] = error_counts.get(message.error, 0) + 1 - else: - await mock_send_to_processors(message.records) - - # Should not send to processors - mock_send_to_processors.assert_not_called() - - # Error should be tracked - assert error in error_counts - assert error_counts[error] == 1 + tracked = ErrorDetails.from_exception(dispatch_error) + assert manager._telemetry_state.error_counts[tracked] == 1 @pytest.mark.asyncio - async def test_send_telemetry_to_results_processors(self): - """Test sending telemetry records to processors.""" - from unittest.mock import AsyncMock, Mock - - from aiperf.common.models import TelemetryMetrics, TelemetryRecord - - # Create mock telemetry processor - mock_processor = Mock() - mock_processor.process_telemetry_record = AsyncMock() + async def test_on_telemetry_records_invalid_tracks_error(self) -> None: + manager = RecordsManager.__new__(RecordsManager) + manager._telemetry_state = ErrorTrackingState() + manager._dispatch_record = AsyncMock(return_value=[]) + error = ErrorDetails(message="Test error", code=500) - records = [ - TelemetryRecord( - timestamp_ns=1000000, + await manager._on_telemetry_records( + TelemetryRecordsMessage( + service_id="test_service", + collector_id="test_collector", dcgm_url="http://localhost:9400/metrics", - gpu_index=0, - gpu_uuid="GPU-123", - gpu_model_name="Test GPU", - telemetry_data=TelemetryMetrics(), - ), - TelemetryRecord( - timestamp_ns=1000001, - dcgm_url="http://localhost:9400/metrics", - gpu_index=1, - gpu_uuid="GPU-456", - gpu_model_name="Test GPU", - telemetry_data=TelemetryMetrics(), - ), - ] - - # Test the logic: each record should be sent to processor - for record in records: - await mock_processor.process_telemetry_record(record) - - # Processor should be called for each record - assert mock_processor.process_telemetry_record.call_count == len(records) - - def test_telemetry_hierarchy_add_record(self): - """Test that telemetry hierarchy adds records correctly.""" - from aiperf.common.models import ( - TelemetryHierarchy, - TelemetryMetrics, - TelemetryRecord, - ) - - hierarchy = TelemetryHierarchy() - - record = TelemetryRecord( - timestamp_ns=1000000, - dcgm_url="http://localhost:9400/metrics", - gpu_index=0, - gpu_uuid="GPU-123", - gpu_model_name="Test GPU", - telemetry_data=TelemetryMetrics( - gpu_power_usage=100.0, - ), + records=[], + error=error, + ) ) - # Add record to hierarchy - hierarchy.add_record(record) - - # Verify hierarchy structure - assert "http://localhost:9400/metrics" in hierarchy.dcgm_endpoints - assert "GPU-123" in hierarchy.dcgm_endpoints["http://localhost:9400/metrics"] + assert manager._telemetry_state.error_counts[error] == 1 + manager._dispatch_record.assert_not_awaited() class TestRecordsManagerTimeslice: - """Test cases for RecordsManager timeslice functionality.""" - - @pytest.mark.asyncio - async def test_process_records_result_with_both_records_and_timeslice(self): - """Test that ProcessRecordsResult can contain both records and timeslice results.""" + """ProfileResults stores accumulator-backed timeslices.""" + + def _timeslices(self, metric_result: MetricResult) -> list[TimesliceResult]: + return [ + TimesliceResult( + start_ns=1_000_000_000, + end_ns=2_000_000_000, + metric_results={metric_result.tag: metric_result}, + ), + TimesliceResult( + start_ns=2_000_000_000, + end_ns=3_000_000_000, + metric_results={metric_result.tag: metric_result}, + ), + ] + def test_process_records_result_with_both_records_and_timeslices(self) -> None: metric_result = MetricResult( tag="request_latency", header="Request Latency", @@ -249,30 +170,22 @@ async def test_process_records_result_with_both_records_and_timeslice(self): count=10, ) - timeslice_results = { - 0: [metric_result], - 1: [metric_result], - } - - # Create a ProcessRecordsResult with both types of results result = ProcessRecordsResult( results=ProfileResults( records=[metric_result, metric_result], - timeslice_metric_results=timeslice_results, + timeslices=self._timeslices(metric_result), completed=2, - start_ns=1000000000, - end_ns=2000000000, + start_ns=1_000_000_000, + end_ns=2_000_000_000, ) ) assert result.results.records is not None assert len(result.results.records) == 2 - assert result.results.timeslice_metric_results is not None - assert len(result.results.timeslice_metric_results) == 2 + assert result.results.timeslices is not None + assert len(result.results.timeslices) == 2 - @pytest.mark.asyncio - async def test_profile_results_serialization_with_timeslice(self): - """Test that ProfileResults with timeslice data can be serialized.""" + def test_profile_results_serialization_with_timeslices(self) -> None: metric_result = MetricResult( tag="request_latency", header="Request Latency", @@ -280,28 +193,20 @@ async def test_profile_results_serialization_with_timeslice(self): avg=100.0, count=10, ) - - timeslice_results = { - 0: [metric_result], - 1: [metric_result], - } - profile_results = ProfileResults( records=[metric_result], - timeslice_metric_results=timeslice_results, + timeslices=self._timeslices(metric_result), completed=1, - start_ns=1000000000, - end_ns=2000000000, + start_ns=1_000_000_000, + end_ns=2_000_000_000, ) - # Test that it can be converted to dict (for JSON serialization) result_dict = profile_results.model_dump() assert "records" in result_dict - assert "timeslice_metric_results" in result_dict - assert result_dict["timeslice_metric_results"] is not None - assert 0 in result_dict["timeslice_metric_results"] - assert 1 in result_dict["timeslice_metric_results"] + assert "timeslices" in result_dict + assert "timeslice_metric_results" not in result_dict + assert len(result_dict["timeslices"]) == 2 def _create_credit_phase_stats() -> CreditPhaseStats: @@ -326,7 +231,6 @@ def _create_credit_phase_stats() -> CreditPhaseStats: def _create_manager_for_timing_dispatch() -> RecordsManager: manager = RecordsManager.__new__(RecordsManager) - # These tests exercise the post-configuration flow; release the barrier. manager._dataset_configured_event = asyncio.Event() manager._dataset_configured_event.set() manager._records_tracker = MagicMock() @@ -334,20 +238,12 @@ def _create_manager_for_timing_dispatch() -> RecordsManager: manager._complete_credit_phases = set() manager._phase_branch_stats = {} manager._latest_branch_stats = None - manager._timing_results_processors = [] - manager._send_timing_to_results_processors = AsyncMock() - manager._send_results_to_results_processors = AsyncMock() - # Accumulator engine dispatch (primary summary path) — stubbed; these tests - # exercise the finalization-ordering logic, not the per-record fan-out. - manager._metric_record_accumulators = [] - manager._metric_record_stream_exporters = [] - manager._send_record_to_accumulators = AsyncMock() + manager._dispatch_record = AsyncMock(return_value=[]) manager._maybe_hint_missing_cache_reporting = MagicMock() manager.info = MagicMock() manager.notice = MagicMock() manager.debug = MagicMock() manager.trace = MagicMock() - manager.exception = MagicMock() manager.is_enabled_for = MagicMock(return_value=False) manager._handle_all_records_received = AsyncMock() return manager @@ -374,7 +270,7 @@ def _metric_records_message( class TestRecordsManagerTimingDispatch: @pytest.mark.asyncio - async def test_on_credit_phase_start_forwards_timing_snapshot(self) -> None: + async def test_on_credit_phase_start_dispatches_timing_snapshot(self) -> None: manager = _create_manager_for_timing_dispatch() stats = _create_credit_phase_stats() message = CreditPhaseStartMessage( @@ -389,54 +285,57 @@ async def test_on_credit_phase_start_forwards_timing_snapshot(self) -> None: await manager._on_credit_phase_start(message) manager._records_tracker.update_phase_info.assert_called_once_with(stats) - manager._send_timing_to_results_processors.assert_awaited_once_with(stats) + manager._dispatch_record.assert_awaited_once_with(stats) @pytest.mark.asyncio - async def test_on_credit_phase_progress_forwards_timing_snapshot(self) -> None: + async def test_on_credit_phase_progress_dispatches_timing_snapshot(self) -> None: manager = _create_manager_for_timing_dispatch() stats = _create_credit_phase_stats() - message = CreditPhaseProgressMessage(service_id="timing-manager", stats=stats) - await manager._on_credit_phase_progress(message) + await manager._on_credit_phase_progress( + CreditPhaseProgressMessage(service_id="timing-manager", stats=stats) + ) manager._records_tracker.update_phase_info.assert_called_once_with(stats) - manager._send_timing_to_results_processors.assert_awaited_once_with(stats) + manager._dispatch_record.assert_awaited_once_with(stats) @pytest.mark.asyncio - async def test_on_credit_phase_sending_complete_forwards_timing_snapshot( + async def test_on_credit_phase_sending_complete_dispatches_timing_snapshot( self, ) -> None: manager = _create_manager_for_timing_dispatch() stats = _create_credit_phase_stats().model_copy( update={"final_requests_sent": 64} ) - message = CreditPhaseSendingCompleteMessage( - service_id="timing-manager", - stats=stats, - ) - await manager._on_credit_phase_sending_complete(message) + await manager._on_credit_phase_sending_complete( + CreditPhaseSendingCompleteMessage( + service_id="timing-manager", + stats=stats, + ) + ) manager._records_tracker.update_phase_info.assert_called_once_with(stats) - manager._send_timing_to_results_processors.assert_awaited_once_with(stats) + manager._dispatch_record.assert_awaited_once_with(stats) @pytest.mark.asyncio - async def test_on_credit_phase_complete_forwards_timing_snapshot(self) -> None: + async def test_on_credit_phase_complete_dispatches_timing_snapshot(self) -> None: manager = _create_manager_for_timing_dispatch() stats = _create_credit_phase_stats().model_copy( update={"final_requests_completed": 64} ) - message = CreditPhaseCompleteMessage(service_id="timing-manager", stats=stats) manager._records_tracker.check_and_set_all_records_received_for_phase.return_value = False manager._records_tracker.create_stats_for_phase.return_value = MagicMock( total_records=64, final_requests_completed=64, ) - await manager._on_credit_phase_complete(message) + await manager._on_credit_phase_complete( + CreditPhaseCompleteMessage(service_id="timing-manager", stats=stats) + ) manager._records_tracker.update_phase_info.assert_called_once_with(stats) - manager._send_timing_to_results_processors.assert_awaited_once_with(stats) + manager._dispatch_record.assert_awaited_once_with(stats) @pytest.mark.asyncio async def test_on_metric_records_records_complete_before_phase_complete_defers_finalization( @@ -591,21 +490,21 @@ async def test_finalization_runs_once_for_all_terminal_event_orders( ) @pytest.mark.asyncio - async def test_finalization_runs_when_final_record_arrives_during_phase_complete_timing_fanout( + async def test_finalization_runs_when_final_record_arrives_during_phase_complete_dispatch( self, ) -> None: manager = _create_manager_for_timing_dispatch() manager._records_tracker = RecordsTracker() - timing_fanout_started = asyncio.Event() - release_timing_fanout = asyncio.Event() + timing_dispatch_started = asyncio.Event() + release_timing_dispatch = asyncio.Event() - async def _block_timing_fanout(stats: CreditPhaseStats) -> None: - timing_fanout_started.set() - await release_timing_fanout.wait() + async def _block_timing_dispatch(record) -> list[BaseException]: + if isinstance(record, CreditPhaseStats): + timing_dispatch_started.set() + await release_timing_dispatch.wait() + return [] - manager._send_timing_to_results_processors = AsyncMock( - side_effect=_block_timing_fanout - ) + manager._dispatch_record = AsyncMock(side_effect=_block_timing_dispatch) phase_complete_task = asyncio.create_task( manager._on_credit_phase_complete( CreditPhaseCompleteMessage( @@ -616,12 +515,12 @@ async def _block_timing_fanout(stats: CreditPhaseStats) -> None: ) ) ) - await timing_fanout_started.wait() + await timing_dispatch_started.wait() await manager._on_metric_records(_metric_records_message()) manager._handle_all_records_received.assert_not_awaited() - release_timing_fanout.set() + release_timing_dispatch.set() await phase_complete_task manager._handle_all_records_received.assert_awaited_once_with( @@ -629,79 +528,17 @@ async def _block_timing_fanout(stats: CreditPhaseStats) -> None: ) @pytest.mark.asyncio - async def test_send_timing_to_results_processors_ignores_empty_processor_list( - self, - ) -> None: - manager = RecordsManager.__new__(RecordsManager) - manager._timing_results_processors = [] - manager.exception = MagicMock() - - await manager._send_timing_to_results_processors(_create_credit_phase_stats()) - - manager.exception.assert_not_called() - - @pytest.mark.asyncio - async def test_send_timing_to_results_processors_swallows_best_effort_failures( - self, - ) -> None: - """Best-effort timing processors (OTel streaming) log but do not re-raise.""" - manager = RecordsManager.__new__(RecordsManager) - ok_processor = MagicMock() - ok_processor.process_result = AsyncMock(return_value=None) - ok_processor.is_best_effort = True - failing_processor = MagicMock() - failing_processor.process_result = AsyncMock( - side_effect=RuntimeError("timing failure") - ) - failing_processor.is_best_effort = True - manager._timing_results_processors = [ok_processor, failing_processor] - manager.exception = MagicMock() - - await manager._send_timing_to_results_processors(_create_credit_phase_stats()) - - ok_processor.process_result.assert_awaited_once() - failing_processor.process_result.assert_awaited_once() - manager.exception.assert_called_once() - - @pytest.mark.asyncio - async def test_send_timing_to_results_processors_reraises_non_best_effort_failures( - self, - ) -> None: - """Non-best-effort timing processors re-raise so bugs surface.""" - manager = RecordsManager.__new__(RecordsManager) - failing_processor = MagicMock() - failing_processor.process_result = AsyncMock( - side_effect=RuntimeError("strict timing failure") - ) - failing_processor.is_best_effort = False - manager._timing_results_processors = [failing_processor] - manager.exception = MagicMock() - - with pytest.raises(RuntimeError, match="strict timing failure"): - await manager._send_timing_to_results_processors( - _create_credit_phase_stats() - ) - - failing_processor.process_result.assert_awaited_once() - manager.exception.assert_called_once() - - @pytest.mark.asyncio - async def test_on_metric_records_processor_raises_still_updates_tracker_and_converges_barrier( + async def test_dispatch_errors_still_update_tracker_and_converge_barrier( self, ) -> None: - """A non-best-effort results processor that raises must not skip the - tracker update or completion-barrier check, or the phase never converges - and the timeout-less barrier hangs. The raise re-propagates after the - finally runs the barrier logic.""" manager = _create_manager_for_timing_dispatch() - manager._send_results_to_results_processors = AsyncMock( - side_effect=RuntimeError("non-best-effort processor exploded") + manager._dispatch_record = AsyncMock( + return_value=[RuntimeError("handler boom")] ) manager._complete_credit_phases = {CreditPhase.PROFILING} manager._records_tracker.check_and_set_all_records_received_for_phase.return_value = True - with pytest.raises(RuntimeError, match="non-best-effort processor exploded"): - await manager._on_metric_records(_metric_records_message()) + await manager._on_metric_records(_metric_records_message()) manager._records_tracker.update_from_record_data.assert_called_once() manager._records_tracker.check_and_set_all_records_received_for_phase.assert_called_once_with( @@ -712,174 +549,8 @@ async def test_on_metric_records_processor_raises_still_updates_tracker_and_conv ) -class TestRecordsManagerProcessorDispatch: - @pytest.mark.asyncio - async def test_send_metric_results_to_results_processors_ignores_empty_processor_list( - self, - ) -> None: - manager = RecordsManager.__new__(RecordsManager) - manager._metric_results_processors = [] - manager.exception = MagicMock() - - await manager._send_results_to_results_processors( - create_metric_record_data(1_000, 2_000) - ) - - manager.exception.assert_not_called() - - @pytest.mark.asyncio - async def test_send_results_to_results_processors_reraises_non_streaming_failures( - self, - ) -> None: - manager = RecordsManager.__new__(RecordsManager) - ok_processor = MagicMock() - ok_processor.process_result = AsyncMock(return_value=None) - ok_processor.is_best_effort = False - failing_processor = MagicMock() - failing_processor.process_result = AsyncMock( - side_effect=RuntimeError("metric processing failed") - ) - failing_processor.is_best_effort = False - manager._metric_results_processors = [ok_processor, failing_processor] - manager.exception = MagicMock() - - with pytest.raises(RuntimeError, match="metric processing failed"): - await manager._send_results_to_results_processors( - create_metric_record_data(1_000, 2_000) - ) - - ok_processor.process_result.assert_awaited_once() - failing_processor.process_result.assert_awaited_once() - manager.exception.assert_called_once() - - @pytest.mark.asyncio - async def test_send_results_to_results_processors_swallows_streaming_failures( - self, - ) -> None: - manager = RecordsManager.__new__(RecordsManager) - ok_processor = MagicMock() - ok_processor.process_result = AsyncMock(return_value=None) - ok_processor.is_best_effort = False - # Streaming processor with is_best_effort=True should be swallowed. - streaming_processor = MagicMock() - streaming_processor.process_result = AsyncMock( - side_effect=RuntimeError("otel fanout failure") - ) - streaming_processor.is_best_effort = True - manager._metric_results_processors = [ok_processor, streaming_processor] - manager.exception = MagicMock() - - # Should NOT raise — streaming processors are best-effort. - await manager._send_results_to_results_processors( - create_metric_record_data(1_000, 2_000) - ) - - ok_processor.process_result.assert_awaited_once() - streaming_processor.process_result.assert_awaited_once() - manager.exception.assert_called_once() - - @pytest.mark.asyncio - async def test_flush_metric_results_processors_flushes_only_flushable(self) -> None: - manager = RecordsManager.__new__(RecordsManager) - manager.exception = MagicMock() - manager.debug = MagicMock() - - class FakeFlushProtocol: - pass - - class FakeFlushable(FakeFlushProtocol): - def __init__(self) -> None: - self.flush = AsyncMock(return_value=None) - - flushable = FakeFlushable() - non_flushable = MagicMock() - manager._metric_results_processors = [flushable, non_flushable] - - with patch( - "aiperf.records.records_manager.FlushableResultsProcessorProtocol", - FakeFlushProtocol, - ): - await manager._flush_metric_results_processors(force=True) - - flushable.flush.assert_awaited_once_with(force=True) - manager.exception.assert_not_called() - - @pytest.mark.asyncio - async def test_flush_metric_results_processors_swallows_best_effort_failures( - self, - ) -> None: - """Best-effort flushable processors (telemetry) log but do not re-raise.""" - manager = RecordsManager.__new__(RecordsManager) - manager.exception = MagicMock() - manager.debug = MagicMock() - - class FakeFlushProtocol: - pass - - class FakeBestEffortFlushable(FakeFlushProtocol): - is_best_effort: bool = True - - def __init__(self) -> None: - self.flush = AsyncMock(side_effect=RuntimeError("otel flush failed")) - - flushable = FakeBestEffortFlushable() - manager._metric_results_processors = [flushable] - - with patch( - "aiperf.records.records_manager.FlushableResultsProcessorProtocol", - FakeFlushProtocol, - ): - # Should NOT raise — best-effort contract. - await manager._flush_metric_results_processors(force=True) - - flushable.flush.assert_awaited_once_with(force=True) - manager.exception.assert_called_once() - - @pytest.mark.asyncio - async def test_flush_metric_results_processors_reraises_non_best_effort_failures( - self, - ) -> None: - """Non-best-effort flushable processors re-raise to surface data-pipeline bugs.""" - manager = RecordsManager.__new__(RecordsManager) - manager.exception = MagicMock() - manager.debug = MagicMock() - - class FakeFlushProtocol: - pass - - class FakeStrictFlushable(FakeFlushProtocol): - is_best_effort: bool = False - - def __init__(self) -> None: - self.flush = AsyncMock( - side_effect=RuntimeError("pipeline flush failed") - ) - - flushable = FakeStrictFlushable() - manager._metric_results_processors = [flushable] - - with ( - patch( - "aiperf.records.records_manager.FlushableResultsProcessorProtocol", - FakeFlushProtocol, - ), - pytest.raises(RuntimeError, match="pipeline flush failed"), - ): - await manager._flush_metric_results_processors(force=True) - - flushable.flush.assert_awaited_once_with(force=True) - manager.exception.assert_called_once() - - class TestRecordsManagerEfficiencyMetricsSnapshot: - """Pin the invariant that `completed` counts request-derived records only. - - `_process_results` snapshots `len(records_results)` BEFORE extending it - with `compute_efficiency_metrics` output. If the snapshot is moved or - the extend is reordered, `completed` would silently bump by the number - of derived aggregates emitted (currently up to 3: total_gpu_power, - total_gpu_energy, output_tokens_per_joule). - """ + """Pin the invariant that `completed` counts request-derived records only.""" @pytest.mark.asyncio async def test_completed_excludes_efficiency_metrics(self) -> None: @@ -889,10 +560,8 @@ async def test_completed_excludes_efficiency_metrics(self) -> None: manager.info = MagicMock() manager.error = MagicMock() manager.exception = MagicMock() - manager.is_enabled_for = MagicMock(return_value=False) manager.service_id = "records-manager-test" manager._latest_branch_stats = None - manager._flush_metric_results_processors = AsyncMock() manager.publish = AsyncMock() manager.run = MagicMock() @@ -904,9 +573,6 @@ async def test_completed_excludes_efficiency_metrics(self) -> None: MetricResult(tag="request_latency", header="h", unit="ms", avg=1.0), MetricResult(tag="output_token_count", header="h", unit="tokens", avg=2.0), ] - # The byte-exact summary engine now sources records from the - # metric_record accumulators (AccumulatorMetricsSummary shape), not the - # legacy MetricResultsProcessor. metric_accumulator = MagicMock() metric_accumulator.summarize = AsyncMock( return_value=AccumulatorMetricsSummary( @@ -942,12 +608,10 @@ async def test_completed_excludes_efficiency_metrics(self) -> None: result = await manager._process_results(CreditPhase.PROFILING, cancelled=False) - assert result.results.completed == len(request_records), ( - "completed must reflect request-derived records only, not derived aggregates" - ) + assert result.results.completed == len(request_records) assert len(result.results.records) == len(request_records) + len( efficiency_metrics - ), "records should include both request-derived and efficiency aggregates" + ) assert {r.tag for r in result.results.records} == { "request_latency", "output_token_count", @@ -958,16 +622,7 @@ async def test_completed_excludes_efficiency_metrics(self) -> None: class TestRecordsManagerEfficiencyMetricsDegeneratePhase: - """Pin the degenerate "no records flowed" guard around the efficiency block. - - When phase_stats.start_ns or requests_end_ns is None (the phase has no - record-derived window), constructing a TimeRangeFilter via two - consecutive time.time_ns() fallbacks would yield an effectively - zero-width window. Power (a gauge) would then either emit a misleading - 0.0W result or be silently dropped depending on telemetry sample jitter. - The guard must skip the efficiency-metrics block entirely and log a - warning naming the phase. - """ + """Efficiency metrics need a real record-derived phase window.""" @pytest.mark.asyncio async def test_none_phase_window_skips_efficiency_metrics_with_warning( @@ -980,10 +635,8 @@ async def test_none_phase_window_skips_efficiency_metrics_with_warning( manager.warning = MagicMock() manager.error = MagicMock() manager.exception = MagicMock() - manager.is_enabled_for = MagicMock(return_value=False) manager.service_id = "records-manager-test" manager._latest_branch_stats = None - manager._flush_metric_results_processors = AsyncMock() manager.publish = AsyncMock() manager.run = MagicMock() @@ -1030,12 +683,10 @@ async def test_none_phase_window_skips_efficiency_metrics_with_warning( assert "Skipping efficiency metrics" in warning_msg assert "start_ns=None" in warning_msg assert "requests_end_ns=None" in warning_msg - assert {r.tag for r in result.results.records} == {"request_latency"} @pytest.mark.asyncio async def test_partial_none_phase_window_also_skips(self) -> None: - """start_ns set but requests_end_ns None must also skip (and vice versa).""" manager = RecordsManager.__new__(RecordsManager) manager.debug = MagicMock() @@ -1043,10 +694,8 @@ async def test_partial_none_phase_window_also_skips(self) -> None: manager.warning = MagicMock() manager.error = MagicMock() manager.exception = MagicMock() - manager.is_enabled_for = MagicMock(return_value=False) manager.service_id = "records-manager-test" manager._latest_branch_stats = None - manager._flush_metric_results_processors = AsyncMock() manager.publish = AsyncMock() manager.run = MagicMock() @@ -1081,64 +730,23 @@ async def test_partial_none_phase_window_also_skips(self) -> None: manager.warning.assert_called_once() -class TestRecordsManagerInitialization: - def test_otel_post_processor_disabled_logs_info( - self, - benchmark_run, - ) -> None: - def _fake_pull_client_init(self, run, **kwargs) -> None: - self.run = run - self.cfg = run.cfg - self.service_id = kwargs.get("service_id") or "records_manager" - self.pub_client = MagicMock() - self.attach_child_lifecycle = MagicMock() - self.debug = MagicMock() - self.info = MagicMock() - self.error = MagicMock() - self.exception = MagicMock() - - class DisabledProcessor: - def __init__(self, **kwargs) -> None: - raise PostProcessorDisabled("disabled for test") - - with ( - patch( - "aiperf.records.records_manager.PullClientMixin.__init__", - new=_fake_pull_client_init, - ), - mock_plugin( - "results_processor", - "otel_metrics_streamer", - DisabledProcessor, - ), - ): - manager = RecordsManager(run=benchmark_run) - - info_messages = [args[0] for args, _ in manager.info.call_args_list] - assert any( - "OTel metrics streamer is disabled and will not be used" in message - for message in info_messages - ) - - class TestMidRunCacheReportingHint: - """RecordsManager warns once, mid-run, when token usage is reported but no - prompt-cache read tokens appear in the streamed records (detection logic - itself is covered in tests/unit/metrics/test_cache_reporting_hint.py).""" + """RecordsManager warns once when usage lacks prompt-cache read tokens.""" def _manager(self) -> RecordsManager: manager = RecordsManager.__new__(RecordsManager) manager.warning = MagicMock() + manager._warned_missing_cache_reporting = False return manager - def test_warns_once_on_first_qualifying_record(self): + def test_warns_once_on_first_qualifying_record(self) -> None: manager = self._manager() record_data = SimpleNamespace(metrics={"usage_prompt_tokens": 1024}) manager._maybe_hint_missing_cache_reporting(record_data) - manager._maybe_hint_missing_cache_reporting(record_data) # a later record + manager._maybe_hint_missing_cache_reporting(record_data) manager.warning.assert_called_once_with(CACHE_REPORTING_HINT) - def test_no_warning_when_cache_reported(self): + def test_no_warning_when_cache_reported(self) -> None: manager = self._manager() record_data = SimpleNamespace( metrics={"usage_prompt_tokens": 1024, "usage_prompt_cache_read_tokens": 0} @@ -1146,7 +754,7 @@ def test_no_warning_when_cache_reported(self): manager._maybe_hint_missing_cache_reporting(record_data) manager.warning.assert_not_called() - def test_no_warning_when_usage_absent(self): + def test_no_warning_when_usage_absent(self) -> None: manager = self._manager() record_data = SimpleNamespace(metrics={"output_sequence_length": 32}) manager._maybe_hint_missing_cache_reporting(record_data) @@ -1154,285 +762,112 @@ def test_no_warning_when_usage_absent(self): class TestRealtimeUpdateGate: - """The realtime block must re-render when EITHER the record count OR the - live server-metrics snapshot changes. The port had gated on record count - alone, freezing the server-metrics row (cache hit rate, KV usage, queue - depth) during lulls where the count was momentarily static.""" - def _manager(self) -> RecordsManager: manager = RecordsManager.__new__(RecordsManager) manager._previous_realtime_records = None manager._previous_realtime_server_snapshot = None return manager - def test_first_tick_is_an_update(self): + def test_first_tick_is_an_update(self) -> None: m = self._manager() assert m._has_realtime_update(0, {}) is True - def test_record_count_change_triggers_update(self): + def test_record_count_change_triggers_update(self) -> None: m = self._manager() m._previous_realtime_records = 10 m._previous_realtime_server_snapshot = {"kv_cache_usage_pct": 50.0} assert m._has_realtime_update(11, {"kv_cache_usage_pct": 50.0}) is True - def test_server_metric_change_triggers_update_even_with_static_records(self): + def test_server_metric_change_triggers_update_even_with_static_records( + self, + ) -> None: m = self._manager() m._previous_realtime_records = 10 m._previous_realtime_server_snapshot = {"kv_cache_usage_pct": 50.0} - # Record count unchanged, but KV usage moved -> must still re-render. assert m._has_realtime_update(10, {"kv_cache_usage_pct": 72.0}) is True - def test_no_change_skips_update(self): + def test_no_change_skips_update(self) -> None: m = self._manager() m._previous_realtime_records = 10 m._previous_realtime_server_snapshot = {"kv_cache_usage_pct": 50.0} assert m._has_realtime_update(10, {"kv_cache_usage_pct": 50.0}) is False -class TestRecordsManagerDatasetConfiguredBarrier: - """The records manager must not run metric records through its results - processors until the DatasetConfiguredNotification has been applied. +class _DatasetAwareHandler: + def __init__(self) -> None: + self.metadata = None - Metric records (PULL socket) and the notification (SUB socket) arrive on - independent channels with no ordering guarantee, so processing must block - on an explicit barrier that _on_dataset_configured releases. - """ + def on_dataset_configured(self, metadata) -> None: + self.metadata = metadata + +class TestRecordsManagerDatasetConfiguredBarrier: @pytest.mark.asyncio - async def test_on_dataset_configured_sets_event(self): - """_on_dataset_configured must release the barrier once processors are configured.""" - mock_self = MagicMock(spec=RecordsManager) - mock_self._dataset_configured_event = asyncio.Event() - mock_self._metric_results_processors = [] - mock_self._accumulators = {} + async def test_on_dataset_configured_sets_event_and_notifies_handlers(self) -> None: + manager = RecordsManager.__new__(RecordsManager) + manager._dataset_configured_event = asyncio.Event() + acc = _DatasetAwareHandler() + exp = _DatasetAwareHandler() + manager._accumulators = {AccumulatorType.METRIC_RESULTS: acc} + manager._stream_exporters = {MagicMock(): exp} + message = MagicMock() + message.metadata = {"task": "accuracy"} - await RecordsManager._on_dataset_configured(mock_self, MagicMock()) + await manager._on_dataset_configured(message) - assert mock_self._dataset_configured_event.is_set() + assert manager._dataset_configured_event.is_set() + assert acc.metadata == message.metadata + assert exp.metadata == message.metadata @pytest.mark.asyncio - async def test_on_metric_records_waits_for_dataset_configured(self): - """_on_metric_records must block until the dataset is configured, then proceed.""" - mock_self = MagicMock(spec=RecordsManager) - mock_self._dataset_configured_event = asyncio.Event() - mock_self.is_trace_enabled = False - # The finally block (F4) always runs the tracker/barrier logic even when - # the processor raises; supply the instance attributes it touches. - mock_self._records_tracker = MagicMock() - mock_self._error_tracker = MagicMock() - mock_self._complete_credit_phases = set() - # First downstream step after the barrier; raising proves the barrier was passed. - mock_self._send_results_to_results_processors = AsyncMock( + async def test_on_metric_records_waits_for_dataset_configured(self) -> None: + manager = RecordsManager.__new__(RecordsManager) + manager._dataset_configured_event = asyncio.Event() + manager.is_enabled_for = MagicMock(return_value=False) + manager._records_tracker = MagicMock() + manager._error_tracker = MagicMock() + manager._complete_credit_phases = set() + manager._maybe_hint_missing_cache_reporting = MagicMock() + manager._dispatch_record = AsyncMock( side_effect=RuntimeError("REACHED_PROCESSING") ) - message = MagicMock() - message.metadata.benchmark_phase = CreditPhase.PROFILING + message = _metric_records_message() - task = asyncio.create_task( - RecordsManager._on_metric_records(mock_self, message) - ) + task = asyncio.create_task(manager._on_metric_records(message)) for _ in range(3): await asyncio.sleep(0) - # Barrier not released: processing has not started. assert not task.done() - assert not mock_self._send_results_to_results_processors.called + manager._dispatch_record.assert_not_called() - # Barrier released: processing proceeds past the wait. - mock_self._dataset_configured_event.set() + manager._dataset_configured_event.set() with pytest.raises(RuntimeError, match="REACHED_PROCESSING"): await asyncio.wait_for(task, timeout=1.0) @pytest.mark.asyncio - async def test_on_metric_records_fails_run_on_config_timeout(self, monkeypatch): - """On dataset-config timeout, abort the run (report error + kill) rather - than process the record without a configured dataset.""" - mock_self = MagicMock(spec=RecordsManager) - mock_self.service_id = "rm-test" - mock_self._dataset_configured_event = asyncio.Event() - mock_self.is_trace_enabled = False - mock_self.publish = AsyncMock() - mock_self._kill = AsyncMock() - mock_self._send_results_to_results_processors = AsyncMock() - message = MagicMock() - message.metadata.benchmark_phase = CreditPhase.PROFILING + async def test_on_metric_records_fails_run_on_config_timeout( + self, monkeypatch + ) -> None: + manager = RecordsManager.__new__(RecordsManager) + manager.service_id = "rm-test" + manager._dataset_configured_event = asyncio.Event() + manager.is_enabled_for = MagicMock(return_value=False) + manager.publish = AsyncMock() + manager._kill = AsyncMock() + manager._dispatch_record = AsyncMock() + message = _metric_records_message() async def _raise_timeout(coro, *args, **kwargs): - coro.close() # avoid "coroutine was never awaited" warning + coro.close() raise TimeoutError monkeypatch.setattr( "aiperf.records.dataset_gate.asyncio.wait_for", _raise_timeout ) - await RecordsManager._on_metric_records(mock_self, message) + await manager._on_metric_records(message) - # Run is failed loudly ... - mock_self._kill.assert_awaited_once() - published = mock_self.publish.await_args.args[0] + manager._kill.assert_awaited_once() + published = manager.publish.await_args.args[0] assert isinstance(published, BaseServiceErrorMessage) - # ... and the record is not processed. - mock_self._send_results_to_results_processors.assert_not_called() - - -def _fake_pull_client_init(self, run, **kwargs) -> None: - """Minimal PullClientMixin.__init__ stand-in so RecordsManager.__init__ - can run its plugin-loading + accumulator-gating logic in isolation.""" - self.run = run - self.cfg = run.cfg - self.service_id = kwargs.get("service_id") or "records_manager" - self.pub_client = MagicMock() - self.attach_child_lifecycle = MagicMock() - self.debug = MagicMock() - self.info = MagicMock() - self.error = MagicMock() - self.exception = MagicMock() - - -class TestAccumulatorGateExactType: - """The accumulator gate must remove exactly the built-in - ``MetricResultsProcessor`` by type identity — NOT any processor whose - class merely happens to be NAMED ``MetricResultsProcessor`` (e.g. an - external plugin subclassing the built-in without renaming it).""" - - def test_gate_removes_canonical_metric_results_processor( - self, benchmark_run - ) -> None: - with patch( - "aiperf.records.records_manager.PullClientMixin.__init__", - new=_fake_pull_client_init, - ): - manager = RecordsManager(run=benchmark_run) - - # Gate precondition: the accumulator engine is driving the summary. - assert AccumulatorType.METRIC_RESULTS in manager._accumulators - assert all( - type(p) is not CanonicalMetricResultsProcessor - for p in manager._metric_results_processors - ) - - def test_gate_keeps_same_name_subclass_active(self, benchmark_run) -> None: - class MetricResultsProcessor(CanonicalMetricResultsProcessor): - """Override plugin subclassing the built-in without renaming.""" - - with ( - patch( - "aiperf.records.records_manager.PullClientMixin.__init__", - new=_fake_pull_client_init, - ), - mock_plugin("results_processor", "metric_results", MetricResultsProcessor), - ): - manager = RecordsManager(run=benchmark_run) - - assert AccumulatorType.METRIC_RESULTS in manager._accumulators - survivor_types = [type(p) for p in manager._metric_results_processors] - assert MetricResultsProcessor in survivor_types - - -class _CancellingExporter: - """Stream exporter whose process_record raises CancelledError.""" - - async def process_record(self, record) -> None: - raise asyncio.CancelledError - - -class _FailingExporter: - """Stream exporter whose process_record raises a plain Exception.""" - - async def process_record(self, record) -> None: - raise ValueError("exporter exploded") - - -class _RecordingExporter: - """Stream exporter that records everything it is given.""" - - def __init__(self) -> None: - self.records: list = [] - - async def process_record(self, record) -> None: - self.records.append(record) - - -class TestStreamExporterFanOutErrorHandling: - """The stream-exporter fan-out loops must swallow only ``Exception``: - ``asyncio.CancelledError`` is a ``BaseException`` and must propagate so - task cancellation is never eaten by a best-effort exporter.""" - - def _server_metrics_record(self) -> ServerMetricsRecord: - return ServerMetricsRecord( - endpoint_url="http://localhost:8081/metrics", - timestamp_ns=1_000_000_000, - metrics={}, - ) - - def _telemetry_record(self) -> TelemetryRecord: - return TelemetryRecord( - timestamp_ns=1_000_000, - dcgm_url="http://localhost:9400/metrics", - gpu_index=0, - gpu_uuid="GPU-123", - gpu_model_name="Test GPU", - telemetry_data=TelemetryMetrics(gpu_power_usage=100.0), - ) - - def _manager_for_server_metrics(self) -> RecordsManager: - manager = RecordsManager.__new__(RecordsManager) - manager.error = MagicMock() - manager.exception = MagicMock() - manager._server_metrics_state = ErrorTrackingState() - manager._server_metrics_processors = [] - return manager - - def _manager_for_telemetry(self) -> RecordsManager: - manager = RecordsManager.__new__(RecordsManager) - manager.error = MagicMock() - manager.exception = MagicMock() - manager._telemetry_state = ErrorTrackingState() - manager._gpu_telemetry_processors = [] - return manager - - @pytest.mark.asyncio - async def test_send_server_metrics_cancelled_error_propagates(self) -> None: - manager = self._manager_for_server_metrics() - manager._server_metrics_stream_exporters = [_CancellingExporter()] - - with pytest.raises(asyncio.CancelledError): - await manager._send_server_metrics_to_results_processors( - self._server_metrics_record() - ) - - @pytest.mark.asyncio - async def test_send_server_metrics_exception_swallowed_and_logged(self) -> None: - manager = self._manager_for_server_metrics() - recording = _RecordingExporter() - manager._server_metrics_stream_exporters = [_FailingExporter(), recording] - record = self._server_metrics_record() - - await manager._send_server_metrics_to_results_processors(record) - - manager.error.assert_called_once() - assert "exporter exploded" in manager.error.call_args.args[0] - assert recording.records == [record] - - @pytest.mark.asyncio - async def test_send_telemetry_cancelled_error_propagates(self) -> None: - manager = self._manager_for_telemetry() - manager._gpu_telemetry_stream_exporters = [_CancellingExporter()] - - with pytest.raises(asyncio.CancelledError): - await manager._send_telemetry_to_results_processors( - [self._telemetry_record()] - ) - - @pytest.mark.asyncio - async def test_send_telemetry_exception_swallowed_and_logged(self) -> None: - manager = self._manager_for_telemetry() - recording = _RecordingExporter() - manager._gpu_telemetry_stream_exporters = [_FailingExporter(), recording] - record = self._telemetry_record() - - await manager._send_telemetry_to_results_processors([record]) - - manager.error.assert_called_once() - assert "exporter exploded" in manager.error.call_args.args[0] - assert recording.records == [record] + manager._dispatch_record.assert_not_called() diff --git a/tests/unit/records/test_records_manager_network_latency.py b/tests/unit/records/test_records_manager_network_latency.py index d5eebd0d88..162e30f26f 100644 --- a/tests/unit/records/test_records_manager_network_latency.py +++ b/tests/unit/records/test_records_manager_network_latency.py @@ -1,12 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for the network-latency handler + RTT delivery in RecordsManager. - -Mirrors the ``RecordsManager.__new__(RecordsManager)`` dispatch-test pattern in -``test_records_manager.py``: only the attributes a method touches are populated, -so each method is exercised in isolation without spinning up the full service. -""" +"""Tests for the network-latency handler + RTT delivery in RecordsManager.""" from __future__ import annotations @@ -34,10 +29,11 @@ def _sample(rtt_ns: int = 1_500_000, success: bool = True) -> NetworkLatencySamp class TestOnNetworkLatencyRecords: @pytest.mark.asyncio - async def test_valid_sample_accumulates_and_forwards(self) -> None: + async def test_valid_sample_accumulates_and_dispatches(self) -> None: manager = RecordsManager.__new__(RecordsManager) manager._network_latency_accumulator = MagicMock() - manager._send_network_latency_to_results_processors = AsyncMock() + manager._network_latency_state = ErrorTrackingState() + manager._dispatch_record = AsyncMock(return_value=[]) sample = _sample() message = NetworkLatencyRecordMessage( @@ -47,15 +43,15 @@ async def test_valid_sample_accumulates_and_forwards(self) -> None: await manager._on_network_latency_records(message) manager._network_latency_accumulator.add_sample.assert_called_once_with(sample) - manager._send_network_latency_to_results_processors.assert_awaited_once_with( - sample - ) + manager._dispatch_record.assert_awaited_once_with(sample) + assert manager._network_latency_state.error_counts == {} @pytest.mark.asyncio - async def test_valid_sample_without_accumulator_still_forwards(self) -> None: + async def test_valid_sample_without_accumulator_still_dispatches(self) -> None: manager = RecordsManager.__new__(RecordsManager) manager._network_latency_accumulator = None - manager._send_network_latency_to_results_processors = AsyncMock() + manager._network_latency_state = ErrorTrackingState() + manager._dispatch_record = AsyncMock(return_value=[]) sample = _sample() await manager._on_network_latency_records( @@ -64,18 +60,33 @@ async def test_valid_sample_without_accumulator_still_forwards(self) -> None: ) ) - manager._send_network_latency_to_results_processors.assert_awaited_once_with( - sample + manager._dispatch_record.assert_awaited_once_with(sample) + + @pytest.mark.asyncio + async def test_dispatch_errors_are_tracked(self) -> None: + manager = RecordsManager.__new__(RecordsManager) + manager._network_latency_accumulator = MagicMock() + manager._network_latency_state = ErrorTrackingState() + dispatch_error = RuntimeError("writer failed") + manager._dispatch_record = AsyncMock(return_value=[dispatch_error]) + + await manager._on_network_latency_records( + NetworkLatencyRecordMessage( + service_id="net-mgr", collector_id="localhost:8000", sample=_sample() + ) ) + tracked = ErrorDetails.from_exception(dispatch_error) + assert manager._network_latency_state.error_counts[tracked] == 1 + @pytest.mark.asyncio - async def test_error_message_increments_error_count_and_does_not_forward( + async def test_error_message_increments_error_count_and_does_not_dispatch( self, ) -> None: manager = RecordsManager.__new__(RecordsManager) manager._network_latency_accumulator = MagicMock() manager._network_latency_state = ErrorTrackingState() - manager._send_network_latency_to_results_processors = AsyncMock() + manager._dispatch_record = AsyncMock(return_value=[]) error = ErrorDetails.from_exception(ConnectionRefusedError("refused")) message = NetworkLatencyRecordMessage( @@ -89,143 +100,79 @@ async def test_error_message_increments_error_count_and_does_not_forward( assert manager._network_latency_state.error_counts[error] == 1 manager._network_latency_accumulator.add_sample.assert_not_called() - manager._send_network_latency_to_results_processors.assert_not_awaited() + manager._dispatch_record.assert_not_awaited() -class TestSendNetworkLatencyToResultsProcessors: - @pytest.mark.asyncio - async def test_empty_processor_list_is_noop(self) -> None: - manager = RecordsManager.__new__(RecordsManager) - manager._network_latency_processors = [] - manager.exception = MagicMock() - - await manager._send_network_latency_to_results_processors(_sample()) - - manager.exception.assert_not_called() - - @pytest.mark.asyncio - async def test_forwards_sample_to_each_processor(self) -> None: - manager = RecordsManager.__new__(RecordsManager) - p1 = MagicMock() - p1.process_network_latency_sample = AsyncMock() - p2 = MagicMock() - p2.process_network_latency_sample = AsyncMock() - manager._network_latency_processors = [p1, p2] - manager.exception = MagicMock() - - sample = _sample() - await manager._send_network_latency_to_results_processors(sample) - - p1.process_network_latency_sample.assert_awaited_once_with(sample) - p2.process_network_latency_sample.assert_awaited_once_with(sample) - manager.exception.assert_not_called() - - @pytest.mark.asyncio - async def test_processor_failure_is_tracked_not_raised(self) -> None: - manager = RecordsManager.__new__(RecordsManager) - ok = MagicMock() - ok.process_network_latency_sample = AsyncMock() - failing = MagicMock() - failing.process_network_latency_sample = AsyncMock( - side_effect=RuntimeError("processor boom") - ) - manager._network_latency_processors = [ok, failing] - manager._network_latency_state = ErrorTrackingState() - manager.exception = MagicMock() - - # Must not raise. - await manager._send_network_latency_to_results_processors(_sample()) - - manager.exception.assert_called_once() - assert sum(manager._network_latency_state.error_counts.values()) == 1 - - -class TestDeliverNetworkRttToProcessors: +class TestDeliverNetworkRttToAccumulators: def _make_manager(self, network_cfg) -> RecordsManager: manager = RecordsManager.__new__(RecordsManager) manager.run = SimpleNamespace(cfg=SimpleNamespace(network_latency=network_cfg)) manager.notice = MagicMock() manager.warning = MagicMock() manager._network_latency_accumulator = None - manager._metric_results_processors = [] manager._metric_record_accumulators = [] return manager def test_disabled_is_noop(self) -> None: - processor = MagicMock() + accumulator = MagicMock() manager = self._make_manager(SimpleNamespace(enabled=False, mean_ms=None)) - manager._metric_results_processors = [processor] + manager._metric_record_accumulators = [accumulator] - manager._deliver_network_rtt_to_processors() + manager._deliver_network_rtt_to_accumulators() - processor.set_network_rtt_ns.assert_not_called() + accumulator.set_network_rtt_ns.assert_not_called() manager.notice.assert_not_called() def test_manual_mean_sets_rtt_ns_and_logs_notice(self) -> None: - processor = MagicMock() + accumulator = MagicMock() manager = self._make_manager(SimpleNamespace(enabled=True, mean_ms=2.5)) - manager._metric_results_processors = [processor] + manager._metric_record_accumulators = [accumulator] - manager._deliver_network_rtt_to_processors() + manager._deliver_network_rtt_to_accumulators() - # 2.5 ms -> 2.5e6 ns delivered to the processor. - processor.set_network_rtt_ns.assert_called_once_with(2.5 * 1e6) + accumulator.set_network_rtt_ns.assert_called_once_with(2.5 * 1e6) manager.notice.assert_called_once() def test_measured_mean_from_accumulator_sets_rtt_ns(self) -> None: - processor = MagicMock() + accumulator = MagicMock() manager = self._make_manager(SimpleNamespace(enabled=True, mean_ms=None)) - manager._metric_results_processors = [processor] + manager._metric_record_accumulators = [accumulator] manager._network_latency_accumulator = MagicMock( mean_rtt_ns=1_750_000.0, successful_sample_count=12 ) - manager._deliver_network_rtt_to_processors() + manager._deliver_network_rtt_to_accumulators() - processor.set_network_rtt_ns.assert_called_once_with(1_750_000.0) + accumulator.set_network_rtt_ns.assert_called_once_with(1_750_000.0) manager.notice.assert_called_once() manager.warning.assert_not_called() - def test_metric_accumulator_also_receives_rtt(self) -> None: - """The MetricsAccumulator engine (not just legacy processors) is the - primary consumer — it injects network_adjusted_* in its summarize().""" - accumulator = MagicMock() - manager = self._make_manager(SimpleNamespace(enabled=True, mean_ms=2.5)) - manager._metric_record_accumulators = [accumulator] - - manager._deliver_network_rtt_to_processors() - - accumulator.set_network_rtt_ns.assert_called_once_with(2.5 * 1e6) - def test_zero_successful_samples_warns_and_applies_no_adjustment(self) -> None: - processor = MagicMock() + accumulator = MagicMock() manager = self._make_manager(SimpleNamespace(enabled=True, mean_ms=None)) - manager._metric_results_processors = [processor] + manager._metric_record_accumulators = [accumulator] manager._network_latency_accumulator = MagicMock(mean_rtt_ns=None) - manager._deliver_network_rtt_to_processors() + manager._deliver_network_rtt_to_accumulators() - # No measurable RTT: warn and leave processors at their default (no injection). manager.warning.assert_called_once() - processor.set_network_rtt_ns.assert_not_called() + accumulator.set_network_rtt_ns.assert_not_called() def test_zero_mean_override_is_noop(self) -> None: - # mean_ms=0 would emit network_adjusted_* identical to the raw metrics; skip it. - processor = MagicMock() + accumulator = MagicMock() manager = self._make_manager(SimpleNamespace(enabled=True, mean_ms=0.0)) - manager._metric_results_processors = [processor] + manager._metric_record_accumulators = [accumulator] - manager._deliver_network_rtt_to_processors() + manager._deliver_network_rtt_to_accumulators() - processor.set_network_rtt_ns.assert_not_called() + accumulator.set_network_rtt_ns.assert_not_called() manager.notice.assert_not_called() - def test_processor_without_setter_is_skipped(self) -> None: - # A processor lacking set_network_rtt_ns must not break delivery. - processor = MagicMock(spec=[]) + def test_accumulator_without_setter_is_skipped(self) -> None: + accumulator = MagicMock(spec=[]) manager = self._make_manager(SimpleNamespace(enabled=True, mean_ms=2.5)) - manager._metric_results_processors = [processor] + manager._metric_record_accumulators = [accumulator] + + manager._deliver_network_rtt_to_accumulators() - # Must not raise. - manager._deliver_network_rtt_to_processors() manager.notice.assert_called_once() diff --git a/tests/unit/records/test_records_manager_process_results.py b/tests/unit/records/test_records_manager_process_results.py index 72832105a3..30bfa3ffba 100644 --- a/tests/unit/records/test_records_manager_process_results.py +++ b/tests/unit/records/test_records_manager_process_results.py @@ -10,10 +10,9 @@ The pipeline: -1. ``_summarize_all_accumulators`` runs ``summarize()`` on every loaded - accumulator, buckets the output by shape, and accumulates errors. -2. ``_finalize_stream_exporters`` flushes JSONL writers concurrently. -3. ``build_process_records_result`` assembles a :class:`ProcessRecordsResult`. +1. ``_deliver_network_rtt_to_accumulators`` wires optional RTT calibration. +2. ``_summarize_metric_record_accumulators`` exports metric-record accumulators. +3. ``_finalize_stream_exporters`` flushes JSONL writers concurrently. 4. ``ProcessRecordsResultMessage`` is published. 5. ``ProcessAllResultsMessage`` is published for the SystemController fan-in. """ @@ -83,7 +82,7 @@ def _make_list_accumulator( results: list[MetricResult] | None = None, summarize_exc: BaseException | None = None, ) -> MagicMock: - """Stub for a legacy-shaped accumulator returning ``list[MetricResult]``.""" + """Stub for an accumulator returning ``list[MetricResult]``.""" acc = MagicMock() acc.__class__.__name__ = "StubListAccumulator" if summarize_exc is not None: @@ -128,10 +127,6 @@ def _make_manager_mock( # Branch-stats snapshot (read by _process_results). mgr._latest_branch_stats = None - # Legacy best-effort processors are flushed (force=True) but do not feed - # the summary numbers — stub it out. - mgr._flush_metric_results_processors = AsyncMock() - # Records tracker — drives the time window via PROFILING phase stats. phase_stats = PhaseRecordsStats( phase=CreditPhase.PROFILING, diff --git a/tests/unit/records/test_records_manager_routing.py b/tests/unit/records/test_records_manager_routing.py index 15bf17a593..37c3944861 100644 --- a/tests/unit/records/test_records_manager_routing.py +++ b/tests/unit/records/test_records_manager_routing.py @@ -1,25 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for the static record-type routing infrastructure. - -Two static lookup helpers in ``records_manager_processing`` dispatch -records to accumulators and stream exporters: - -* :func:`accumulators_for_record_type` and - :func:`stream_exporters_for_record_type` — pure functions that read the - ``record_types`` metadata from ``plugins.iter_entries(...)`` and return - the matching accumulator/exporter instances. Called once at - ``RecordsManager.__init__`` time so the hot path is a list iteration, - not a per-record plugin scan. -* ``_send_record_to_accumulators`` — fans a record out to the precomputed - ``_metric_record_accumulators`` and ``_metric_record_stream_exporters`` - lists; per-handler exceptions are caught so one bad handler does not - abort the others. +"""Tests for metadata-driven record routing in ``RecordsManager``. + +``RecordsManager`` builds a ``record_type -> handlers`` table from the +``record_types`` metadata on accumulator and stream-exporter plugins. The hot path +then dispatches each typed record to the handlers for its ``record_type``. """ from __future__ import annotations +import asyncio from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -35,16 +26,14 @@ SummaryContext, ) from aiperf.common.exceptions import PluginDisabled -from aiperf.plugin.enums import AccumulatorType, StreamExporterType +from aiperf.plugin.enums import AccumulatorType, PluginType, StreamExporterType from aiperf.records.records_manager import RecordsManager from aiperf.records.records_manager_processing import ( - accumulators_for_record_type, load_accumulators, - stream_exporters_for_record_type, ) # --------------------------------------------------------------------------- -# Fake plugin entries (k8s plugin metadata shape) +# Fake plugin entries # --------------------------------------------------------------------------- @@ -57,7 +46,7 @@ def _make_entry(name: str, record_types: list[str]) -> MagicMock: # --------------------------------------------------------------------------- -# Stub processors (protocol-conformant) +# Stub handlers (protocol-conformant) # --------------------------------------------------------------------------- @@ -98,238 +87,200 @@ def __init__(self) -> None: self.get_export_info = MagicMock() +def _set_plugin_entries( + monkeypatch: pytest.MonkeyPatch, + *, + accumulator_entries: list[MagicMock] | None = None, + stream_exporter_entries: list[MagicMock] | None = None, +) -> None: + accumulator_entries = accumulator_entries or [] + stream_exporter_entries = stream_exporter_entries or [] + + def _iter_entries(category: PluginType): + if category == PluginType.ACCUMULATOR: + return iter(accumulator_entries) + if category == PluginType.STREAM_EXPORTER: + return iter(stream_exporter_entries) + return iter(()) + + monkeypatch.setattr( + "aiperf.records.records_manager.plugins.iter_entries", + _iter_entries, + ) + + # --------------------------------------------------------------------------- -# Tests: accumulators_for_record_type / stream_exporters_for_record_type +# Tests: _build_routing_table # --------------------------------------------------------------------------- -class TestAccumulatorsForRecordType: - """Static plugin-metadata lookup replaces the source's _routing_table.""" +class TestBuildRoutingTable: + """Plugin metadata controls which handlers receive each record type.""" - def test_single_accumulator_matches_record_type(self, monkeypatch) -> None: + def test_single_accumulator_matches_record_type( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: acc = StubAccumulator() - accs = {AccumulatorType.METRIC_RESULTS: acc} - entries = [_make_entry("metric_results", ["metric_records"])] - - monkeypatch.setattr( - "aiperf.records.records_manager_processing.plugins.iter_entries", - lambda category: iter(entries), + manager = RecordsManager.__new__(RecordsManager) + manager._accumulators = {AccumulatorType.METRIC_RESULTS: acc} + manager._stream_exporters = {} + _set_plugin_entries( + monkeypatch, + accumulator_entries=[_make_entry("metric_results", ["metric_records"])], ) - matched = accumulators_for_record_type(accs, "metric_records") - assert matched == [acc] + table = manager._build_routing_table() - def test_no_match_for_unknown_record_type(self, monkeypatch) -> None: - acc = StubAccumulator() - accs = {AccumulatorType.METRIC_RESULTS: acc} - entries = [_make_entry("metric_results", ["metric_records"])] - - monkeypatch.setattr( - "aiperf.records.records_manager_processing.plugins.iter_entries", - lambda category: iter(entries), - ) + assert table == {"metric_records": [acc]} - matched = accumulators_for_record_type(accs, "telemetry_records") - assert matched == [] - - def test_only_matching_accumulators_returned(self, monkeypatch) -> None: - """Different accumulators register under different record_types.""" - acc_metric = StubAccumulator() - acc_telemetry = StubAccumulator() - accs = { - AccumulatorType.METRIC_RESULTS: acc_metric, - AccumulatorType.GPU_TELEMETRY: acc_telemetry, + def test_only_matching_handlers_are_routed( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + metric_acc = StubAccumulator() + telemetry_acc = StubAccumulator() + exporter = StubStreamExporter() + manager = RecordsManager.__new__(RecordsManager) + manager._accumulators = { + AccumulatorType.METRIC_RESULTS: metric_acc, + AccumulatorType.GPU_TELEMETRY: telemetry_acc, } - entries = [ - _make_entry("metric_results", ["metric_records"]), - _make_entry("gpu_telemetry", ["telemetry_records"]), - ] - - monkeypatch.setattr( - "aiperf.records.records_manager_processing.plugins.iter_entries", - lambda category: iter(entries), + manager._stream_exporters = { + StreamExporterType.RECORD_EXPORT: exporter, + } + _set_plugin_entries( + monkeypatch, + accumulator_entries=[ + _make_entry("metric_results", ["metric_records"]), + _make_entry("gpu_telemetry", ["gpu_telemetry"]), + ], + stream_exporter_entries=[ + _make_entry("record_export", ["metric_records"]), + ], ) - assert accumulators_for_record_type(accs, "metric_records") == [acc_metric] - assert accumulators_for_record_type(accs, "telemetry_records") == [ - acc_telemetry - ] - - def test_skips_entries_not_in_loaded_dict(self, monkeypatch) -> None: - """Entries with no instantiated accumulator (disabled) are skipped.""" - acc = StubAccumulator() - accs = {AccumulatorType.METRIC_RESULTS: acc} - # Two entries declare "metric_records" but only one is loaded. - entries = [ - _make_entry("metric_results", ["metric_records"]), - _make_entry("server_metrics", ["metric_records"]), - ] - - monkeypatch.setattr( - "aiperf.records.records_manager_processing.plugins.iter_entries", - lambda category: iter(entries), - ) + table = manager._build_routing_table() - matched = accumulators_for_record_type(accs, "metric_records") - assert matched == [acc] + assert table["metric_records"] == [metric_acc, exporter] + assert table["gpu_telemetry"] == [telemetry_acc] - def test_empty_accumulators_dict_returns_empty(self, monkeypatch) -> None: - entries = [_make_entry("metric_results", ["metric_records"])] - monkeypatch.setattr( - "aiperf.records.records_manager_processing.plugins.iter_entries", - lambda category: iter(entries), + def test_skips_entries_not_loaded(self, monkeypatch: pytest.MonkeyPatch) -> None: + acc = StubAccumulator() + manager = RecordsManager.__new__(RecordsManager) + manager._accumulators = {AccumulatorType.METRIC_RESULTS: acc} + manager._stream_exporters = {} + _set_plugin_entries( + monkeypatch, + accumulator_entries=[ + _make_entry("metric_results", ["metric_records"]), + _make_entry("server_metrics", ["metric_records"]), + ], ) - assert accumulators_for_record_type({}, "metric_records") == [] + table = manager._build_routing_table() -class TestStreamExportersForRecordType: - def test_single_stream_exporter_matches(self, monkeypatch) -> None: - exp = StubStreamExporter() - exporters = {StreamExporterType.RECORD_EXPORT: exp} - entries = [_make_entry("record_export", ["metric_records"])] + assert table["metric_records"] == [acc] - monkeypatch.setattr( - "aiperf.records.records_manager_processing.plugins.iter_entries", - lambda category: iter(entries), - ) - - matched = stream_exporters_for_record_type(exporters, "metric_records") - assert matched == [exp] - - def test_only_matching_exporters_returned(self, monkeypatch) -> None: - exp_record = StubStreamExporter() - exp_telemetry = StubStreamExporter() - exporters = { - StreamExporterType.RECORD_EXPORT: exp_record, - StreamExporterType.GPU_TELEMETRY_JSONL_WRITER: exp_telemetry, - } - entries = [ - _make_entry("record_export", ["metric_records"]), - _make_entry("gpu_telemetry_jsonl_writer", ["telemetry_records"]), - ] - - monkeypatch.setattr( - "aiperf.records.records_manager_processing.plugins.iter_entries", - lambda category: iter(entries), + def test_empty_loaded_handlers_returns_empty_table( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + manager = RecordsManager.__new__(RecordsManager) + manager._accumulators = {} + manager._stream_exporters = {} + _set_plugin_entries( + monkeypatch, + accumulator_entries=[_make_entry("metric_results", ["metric_records"])], ) - assert stream_exporters_for_record_type(exporters, "metric_records") == [ - exp_record - ] + assert manager._build_routing_table() == {} # --------------------------------------------------------------------------- -# Tests: _send_record_to_accumulators (per-record dispatch hot path) +# Tests: _dispatch_record # --------------------------------------------------------------------------- -def _make_dispatch_manager_mock( - accumulators_list: list[Any], - exporters_list: list[Any], -) -> MagicMock: - """Mock RecordsManager with the precomputed dispatch lists pre-populated. - - Mirrors the source-branch ``_make_manager_mock`` helper but adapts to - K8s's static lookup: ``_metric_record_accumulators`` and - ``_metric_record_stream_exporters`` are computed in ``__init__`` from - ``accumulators_for_record_type`` / ``stream_exporters_for_record_type``, - so we set them directly. - """ - mgr = MagicMock() - mgr._metric_record_accumulators = accumulators_list - mgr._metric_record_stream_exporters = exporters_list - mgr.error = MagicMock() - mgr.warning = MagicMock() - mgr.debug = MagicMock() - mgr._send_record_to_accumulators = ( - RecordsManager._send_record_to_accumulators.__get__(mgr) - ) - return mgr +class TestDispatchRecord: + """Per-record fan-out uses the metadata-derived routing table.""" - -class TestSendRecordToAccumulators: - """Test K8s's per-record fan-out (replaces source's _dispatch_record).""" + def _manager(self, routing_table: dict[str, list[Any]]) -> RecordsManager: + manager = RecordsManager.__new__(RecordsManager) + manager._routing_table = routing_table + manager.error = MagicMock() + manager.debug = MagicMock() + return manager @pytest.mark.asyncio async def test_dispatch_calls_all_handlers(self) -> None: acc = StubAccumulator() exp = StubStreamExporter() - mgr = _make_dispatch_manager_mock([acc], [exp]) + manager = self._manager({"metric_records": [acc, exp]}) + record = MagicMock(record_type="metric_records") - record = MagicMock() - await mgr._send_record_to_accumulators(record) + errors = await manager._dispatch_record(record) - acc.process_record.assert_called_once_with(record) - exp.process_record.assert_called_once_with(record) + assert errors == [] + acc.process_record.assert_awaited_once_with(record) + exp.process_record.assert_awaited_once_with(record) @pytest.mark.asyncio async def test_dispatch_with_no_handlers_is_noop(self) -> None: - """Empty dispatch lists short-circuit — no error, no crash.""" - mgr = _make_dispatch_manager_mock([], []) + manager = self._manager({}) + record = MagicMock(record_type="metric_records") - await mgr._send_record_to_accumulators(MagicMock()) + errors = await manager._dispatch_record(record) - # No errors reported - mgr.error.assert_not_called() + assert errors == [] + manager.debug.assert_called_once() + manager.error.assert_not_called() @pytest.mark.asyncio - async def test_dispatch_handler_exception_logged(self) -> None: - """One handler raising does not prevent other handlers from running.""" + async def test_dispatch_missing_record_type_returns_error(self) -> None: + manager = self._manager({}) + record = object() + + errors = await manager._dispatch_record(record) + + assert len(errors) == 1 + assert isinstance(errors[0], TypeError) + manager.error.assert_called_once() + + @pytest.mark.asyncio + async def test_dispatch_handler_exception_logged_and_returned(self) -> None: acc = StubAccumulator() acc.process_record.side_effect = RuntimeError("boom") exp = StubStreamExporter() - mgr = _make_dispatch_manager_mock([acc], [exp]) + manager = self._manager({"metric_records": [acc, exp]}) + record = MagicMock(record_type="metric_records") - record = MagicMock() - await mgr._send_record_to_accumulators(record) + errors = await manager._dispatch_record(record) - # Exporter should still be called despite accumulator failure - exp.process_record.assert_called_once_with(record) - # Error should be logged - mgr.error.assert_called_once() - assert "boom" in mgr.error.call_args[0][0] + exp.process_record.assert_awaited_once_with(record) + assert len(errors) == 1 + assert isinstance(errors[0], RuntimeError) + manager.error.assert_called_once() + assert "boom" in manager.error.call_args[0][0] @pytest.mark.asyncio async def test_dispatch_multiple_handler_exceptions(self) -> None: - """Multiple handler failures are each logged independently.""" acc = StubAccumulator() acc.process_record.side_effect = RuntimeError("acc error") exp = StubStreamExporter() exp.process_record.side_effect = ValueError("exp error") - mgr = _make_dispatch_manager_mock([acc], [exp]) + manager = self._manager({"metric_records": [acc, exp]}) - await mgr._send_record_to_accumulators(MagicMock()) + errors = await manager._dispatch_record(MagicMock(record_type="metric_records")) - assert mgr.error.call_count == 2 + assert len(errors) == 2 + assert manager.error.call_count == 2 @pytest.mark.asyncio - async def test_handler_order_accumulators_before_exporters(self) -> None: - """Accumulators run before stream exporters in the gather targets.""" - call_order: list[str] = [] - + async def test_cancelled_error_propagates(self) -> None: acc = StubAccumulator() + acc.process_record.side_effect = asyncio.CancelledError + manager = self._manager({"metric_records": [acc]}) - async def acc_record(_record: Any) -> None: - call_order.append("acc") - - acc.process_record.side_effect = acc_record - - exp = StubStreamExporter() - - async def exp_record(_record: Any) -> None: - call_order.append("exp") - - exp.process_record.side_effect = exp_record - - mgr = _make_dispatch_manager_mock([acc], [exp]) - - await mgr._send_record_to_accumulators(MagicMock()) - - # Targets list is [*accumulators, *exporters] — gather may interleave - # but the *targets* list ordering is observable via zip in the error - # path. Both must have run. - assert "acc" in call_order - assert "exp" in call_order + with pytest.raises(asyncio.CancelledError): + await manager._dispatch_record(MagicMock(record_type="metric_records")) # --------------------------------------------------------------------------- @@ -388,7 +339,6 @@ async def test_finalize_calls_all_exporters(self) -> None: async def test_finalize_empty_exporters_noop(self) -> None: mgr = _make_finalize_manager_mock({}) await mgr._finalize_stream_exporters() - # No error, no crash mgr.error.assert_not_called() @pytest.mark.asyncio @@ -406,10 +356,8 @@ async def test_finalize_error_logged_per_exporter(self) -> None: await mgr._finalize_stream_exporters() - # Both should be called (gather runs all concurrently) exp1.finalize.assert_called_once() exp2.finalize.assert_called_once() - # Error logged for the failing one mgr.error.assert_called_once() assert "flush failed" in mgr.error.call_args[0][0] @@ -431,32 +379,6 @@ async def test_finalize_multiple_errors(self) -> None: assert mgr.error.call_count == 2 -# --------------------------------------------------------------------------- -# Source-branch _dispatch_record / _routing_table — intentionally absent -# --------------------------------------------------------------------------- - - -@pytest.mark.skip( - reason="k8s uses static accumulators_for_record_type, not _dispatch_record" -) -def test_dispatch_record_method_exists() -> None: - """Source branch had RecordsManager._dispatch_record. K8s replaced it - with _send_record_to_accumulators driven by precomputed lists set in - __init__ via accumulators_for_record_type / stream_exporters_for_record_type. - See TestSendRecordToAccumulators above for the ported behavior.""" - - -@pytest.mark.skip( - reason="k8s uses static accumulators_for_record_type, not _routing_table" -) -def test_routing_table_attribute_exists() -> None: - """Source branch built RecordsManager._routing_table at init time as - dict[str, list[handler]] keyed by record_type. K8s replaces it with two - precomputed flat lists per record type (just metric_records today). See - TestAccumulatorsForRecordType / TestStreamExportersForRecordType above - for the ported behavior.""" - - # --------------------------------------------------------------------------- # Tests: load_accumulators construction-failure policy # --------------------------------------------------------------------------- @@ -495,7 +417,7 @@ class TestLoadAccumulatorsConstructionFailure: a silent skip.""" def test_load_accumulators_metric_results_construction_failure_reraises( - self, monkeypatch + self, monkeypatch: pytest.MonkeyPatch ) -> None: entries = [_make_entry("metric_results", ["metric_records"])] monkeypatch.setattr( @@ -512,9 +434,9 @@ def test_load_accumulators_metric_results_construction_failure_reraises( load_accumulators(host) def test_load_accumulators_optional_accumulator_failure_swallowed( - self, monkeypatch + self, monkeypatch: pytest.MonkeyPatch ) -> None: - entries = [_make_entry("gpu_telemetry", ["telemetry_records"])] + entries = [_make_entry("gpu_telemetry", ["gpu_telemetry"])] monkeypatch.setattr( "aiperf.records.records_manager_processing.plugins.iter_entries", lambda _plugin_type: entries, @@ -531,7 +453,7 @@ def test_load_accumulators_optional_accumulator_failure_swallowed( host.error.assert_called_once() def test_load_accumulators_metric_results_disabled_is_silent_skip( - self, monkeypatch + self, monkeypatch: pytest.MonkeyPatch ) -> None: entries = [_make_entry("metric_results", ["metric_records"])] monkeypatch.setattr( diff --git a/tests/unit/server_metrics/test_accumulator.py b/tests/unit/server_metrics/test_accumulator.py index e3a398e802..25a2ebfc4e 100644 --- a/tests/unit/server_metrics/test_accumulator.py +++ b/tests/unit/server_metrics/test_accumulator.py @@ -79,8 +79,8 @@ def sample_server_metrics_record( @pytest.mark.asyncio -class TestServerMetricsResultsProcessor: - """Test cases for ServerMetricsResultsProcessor.""" +class TestServerMetricsAccumulator: + """Test cases for ServerMetricsAccumulator.""" async def test_initialization(self, mock_cfg: BenchmarkRun) -> None: """Test processor initialization sets up hierarchy.""" From cd8dbdd1762688d71db0bfde113a3373b7ae5022 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 16:51:39 -0700 Subject: [PATCH 02/39] fix(accuracy): phase-scope accuracy counts so warmup and profiling don't mix AccuracyResultsProcessor is a metric_records accumulator, but it was summarized via the unscoped summarize() while MetricsAccumulator got the phase-scoped export_results(ctx). Because process_record() increments one global counter set for every record, warmup grades leaked into the profiling accuracy summary (and profiling records into any warmup summary): one correct warmup + one incorrect profiling record produced accuracy.overall count=2/current=0.5 instead of the profiling-only count=1/current=0.0. Key the correct/total/unparsed counters by CreditPhase (from record.metadata.benchmark_phase) and add export_results(ctx) that scopes the summary to ctx.phase (summing all phases when None). Mark both MetricsAccumulator and AccuracyResultsProcessor with `supports_phase_scoped_export` and route them through export_results(ctx) in RecordsManager instead of hardcoding the MetricsAccumulator class name. summarize() keeps its phase-agnostic full-range behavior for existing callers. Reported-by: ilana-n Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- .../accuracy/accuracy_results_processor.py | 144 ++++++++++++++---- src/aiperf/metrics/accumulator.py | 4 + src/aiperf/records/records_manager.py | 13 +- .../test_accuracy_record_processor.py | 27 ++-- ...st_accuracy_results_processor_summarize.py | 123 +++++++++++---- 5 files changed, 229 insertions(+), 82 deletions(-) diff --git a/src/aiperf/accuracy/accuracy_results_processor.py b/src/aiperf/accuracy/accuracy_results_processor.py index 88d216680e..a2db4465d8 100644 --- a/src/aiperf/accuracy/accuracy_results_processor.py +++ b/src/aiperf/accuracy/accuracy_results_processor.py @@ -14,12 +14,13 @@ accuracy_task_tag, accuracy_unparsed_task_tag, ) -from aiperf.common.enums import MetricConsoleGroup +from aiperf.common.enums import CreditPhase, MetricConsoleGroup from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.mixins import AIPerfLifecycleMixin from aiperf.common.models import MetricResult if TYPE_CHECKING: + from aiperf.common.accumulator_protocols import ExportContext from aiperf.common.messages.inference_messages import MetricRecordsData from aiperf.common.models.dataset_models import DatasetMetadata from aiperf.config.resolution.plan import BenchmarkRun @@ -32,8 +33,17 @@ class AccuracyResultsProcessor(AIPerfLifecycleMixin): when DatasetConfiguredNotification arrives). Accumulates per-record grading results from AccuracyRecordProcessor, then summarizes into per-task and overall accuracy MetricResult objects. + + Counts are keyed by ``CreditPhase`` so that ``export_results(ctx)`` can scope + a summary to a single phase (e.g. PROFILING), matching how MetricsAccumulator + is phase-scoped. Without this, warmup grades would leak into the profiling + accuracy summary (and vice versa) since every record increments the counters. """ + # RecordsManager routes phase-scoped-export accumulators through + # export_results(ctx) instead of the unscoped summarize(). + supports_phase_scoped_export = True + def __init__(self, run: BenchmarkRun, **kwargs: Any) -> None: acc_cfg = run.cfg.accuracy if acc_cfg is None or not acc_cfg.enabled: @@ -45,12 +55,19 @@ def __init__(self, run: BenchmarkRun, **kwargs: Any) -> None: self.run = run self._tasks: list[str] | None = None - self._task_correct: dict[str, int] = defaultdict(int) - self._task_total: dict[str, int] = defaultdict(int) - self._task_unparsed: dict[str, int] = defaultdict(int) - self._overall_correct: int = 0 - self._overall_total: int = 0 - self._overall_unparsed: int = 0 + # Per-phase counts: phase -> task -> count. Overall is a phase -> count. + self._task_correct: dict[CreditPhase, dict[str, int]] = defaultdict( + lambda: defaultdict(int) + ) + self._task_total: dict[CreditPhase, dict[str, int]] = defaultdict( + lambda: defaultdict(int) + ) + self._task_unparsed: dict[CreditPhase, dict[str, int]] = defaultdict( + lambda: defaultdict(int) + ) + self._overall_correct: dict[CreditPhase, int] = defaultdict(int) + self._overall_total: dict[CreditPhase, int] = defaultdict(int) + self._overall_unparsed: dict[CreditPhase, int] = defaultdict(int) def on_dataset_configured(self, metadata: DatasetMetadata) -> None: """Receive task names from the DatasetConfiguredNotification. @@ -85,24 +102,61 @@ async def process_record(self, record_data: MetricRecordsData) -> None: if correct is None: return + phase = record_data.metadata.benchmark_phase task = self._tasks[record_data.metadata.session_num % len(self._tasks)] is_correct = float(correct) >= 0.5 is_unparsed = float(metrics.get(ACCURACY_RECORD_UNPARSED_KEY, 0.0)) >= 0.5 - self._overall_total += 1 + self._overall_total[phase] += 1 if is_correct: - self._overall_correct += 1 + self._overall_correct[phase] += 1 if is_unparsed: - self._overall_unparsed += 1 + self._overall_unparsed[phase] += 1 - self._task_total[task] += 1 + self._task_total[phase][task] += 1 if is_correct: - self._task_correct[task] += 1 + self._task_correct[phase][task] += 1 if is_unparsed: - self._task_unparsed[task] += 1 + self._task_unparsed[phase][task] += 1 - async def summarize(self) -> list[MetricResult]: - """Return overall and per-task accuracy and unparsed counts as MetricResult list. + def _phase_scoped_counts( + self, phase: CreditPhase | None + ) -> tuple[int, int, int, dict[str, int], dict[str, int], dict[str, int]]: + """Collapse the per-phase counters into a single scope. + + ``phase`` selects one phase; ``None`` sums across every phase (the + phase-agnostic full-range view). Returns + ``(overall_correct, overall_total, overall_unparsed, + task_correct, task_total, task_unparsed)``. + """ + phases = [phase] if phase is not None else list(self._overall_total.keys()) + + overall_correct = sum(self._overall_correct.get(p, 0) for p in phases) + overall_total = sum(self._overall_total.get(p, 0) for p in phases) + overall_unparsed = sum(self._overall_unparsed.get(p, 0) for p in phases) + + task_correct: dict[str, int] = defaultdict(int) + task_total: dict[str, int] = defaultdict(int) + task_unparsed: dict[str, int] = defaultdict(int) + for p in phases: + for task, count in self._task_total.get(p, {}).items(): + task_total[task] += count + for task, count in self._task_correct.get(p, {}).items(): + task_correct[task] += count + for task, count in self._task_unparsed.get(p, {}).items(): + task_unparsed[task] += count + + return ( + overall_correct, + overall_total, + overall_unparsed, + task_correct, + task_total, + task_unparsed, + ) + + def _build_results(self, phase: CreditPhase | None) -> list[MetricResult]: + """Build accuracy MetricResults scoped to ``phase`` (or all phases). Emits: - ``accuracy.overall``: overall correct/total ratio @@ -110,20 +164,28 @@ async def summarize(self) -> list[MetricResult]: - ``accuracy.unparsed``: overall count of responses that required regex fallback - ``accuracy.unparsed.task.``: per-task unparsed counts (sorted alphabetically) - Returns an empty list if no records were processed. + Returns an empty list if no records were processed in scope. """ + ( + overall_correct, + overall_total, + overall_unparsed, + task_correct, + task_total, + task_unparsed, + ) = self._phase_scoped_counts(phase) + results: list[MetricResult] = [] - if self._overall_total > 0: - overall_acc = self._overall_correct / self._overall_total + if overall_total > 0: results.append( MetricResult( tag=ACCURACY_OVERALL_TAG, header="Accuracy (Overall)", unit="ratio", - count=self._overall_total, - current=overall_acc, - sum=self._overall_correct, + count=overall_total, + current=overall_correct / overall_total, + sum=overall_correct, # Rendered by the dedicated Accuracy Benchmark Results table, # not the main LLM metrics table (a ratio has no avg/p99/etc, # so it would show as a row of N/A there). @@ -131,38 +193,37 @@ async def summarize(self) -> list[MetricResult]: ) ) - for task in sorted(self._task_total.keys()): - total = self._task_total[task] - correct = self._task_correct[task] - acc = correct / total if total > 0 else 0.0 + for task in sorted(task_total.keys()): + total = task_total[task] + correct = task_correct.get(task, 0) results.append( MetricResult( tag=accuracy_task_tag(task), header=f"Accuracy ({task})", unit="ratio", count=total, - current=acc, + current=correct / total if total > 0 else 0.0, sum=correct, console_group=MetricConsoleGroup.NONE, ) ) - if self._overall_total > 0: + if overall_total > 0: results.append( MetricResult( tag=ACCURACY_UNPARSED_TAG, header="Accuracy Unparsed (Overall)", unit="ratio", - count=self._overall_total, - current=self._overall_unparsed / self._overall_total, - sum=self._overall_unparsed, + count=overall_total, + current=overall_unparsed / overall_total, + sum=overall_unparsed, console_group=MetricConsoleGroup.NONE, ) ) - for task in sorted(self._task_total.keys()): - total = self._task_total[task] - unparsed = self._task_unparsed.get(task, 0) + for task in sorted(task_total.keys()): + total = task_total[task] + unparsed = task_unparsed.get(task, 0) results.append( MetricResult( tag=accuracy_unparsed_task_tag(task), @@ -176,3 +237,20 @@ async def summarize(self) -> list[MetricResult]: ) return results + + async def summarize(self) -> list[MetricResult]: + """Return phase-agnostic (all-phase) accuracy and unparsed counts. + + Prefer ``export_results(ctx)`` for phase-scoped summaries; this full-range + view is retained for phase-agnostic callers. + """ + return self._build_results(phase=None) + + async def export_results(self, ctx: ExportContext) -> list[MetricResult]: + """Return accuracy counts scoped to ``ctx.phase`` (all phases if None). + + RecordsManager summarizes profiling and warmup separately; scoping here + keeps warmup grades out of the profiling accuracy summary (and vice + versa), matching MetricsAccumulator's phase-scoped export. + """ + return self._build_results(phase=ctx.phase) diff --git a/src/aiperf/metrics/accumulator.py b/src/aiperf/metrics/accumulator.py index c1573a92b2..67dc90f443 100644 --- a/src/aiperf/metrics/accumulator.py +++ b/src/aiperf/metrics/accumulator.py @@ -75,6 +75,10 @@ class MetricsAccumulator(BaseMetricsProcessor): DERIVED metrics computed from those at summarize time. """ + # RecordsManager routes phase-scoped-export accumulators through + # export_results(ctx) so warmup records are excluded from profiling summaries. + supports_phase_scoped_export = True + def __init__( self, run: BenchmarkRun, diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 958b6ab8d5..17a164e6e0 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -1070,15 +1070,18 @@ async def _summarize_one_accumulator( Returns the result (or exception object) so a single bad accumulator cannot abort the rest. Accumulators that support phase/window-scoped - export (MetricsAccumulator) get ``export_results(ctx)`` so warmup records - are excluded from profiling summaries; otherwise prefers ``summarize()`` - and falls back to ``export_results(ctx)``. + export (marked with ``supports_phase_scoped_export`` — MetricsAccumulator + and AccuracyResultsProcessor) get ``export_results(ctx)`` so warmup + records are excluded from profiling summaries; otherwise prefers + ``summarize()`` and falls back to ``export_results(ctx)``. """ name = accumulator.__class__.__name__ self.debug(f"Starting summarize for accumulator {acc_type}: {name}") try: - if accumulator.__class__.__name__ == "MetricsAccumulator" and hasattr( - accumulator, "export_results" + # ``is True`` (not truthiness) so a MagicMock's auto-created attribute + # does not spuriously route mock accumulators through export_results. + if getattr(accumulator, "supports_phase_scoped_export", False) is True and ( + hasattr(accumulator, "export_results") ): res = await asyncio.wait_for( accumulator.export_results(ctx), diff --git a/tests/unit/accuracy/test_accuracy_record_processor.py b/tests/unit/accuracy/test_accuracy_record_processor.py index 198139a289..054327228b 100644 --- a/tests/unit/accuracy/test_accuracy_record_processor.py +++ b/tests/unit/accuracy/test_accuracy_record_processor.py @@ -8,6 +8,7 @@ from aiperf.accuracy.accuracy_record_processor import AccuracyRecordProcessor from aiperf.accuracy.accuracy_results_processor import AccuracyResultsProcessor from aiperf.accuracy.models import GradingResult +from aiperf.common.enums import CreditPhase from aiperf.common.messages.inference_messages import MetricRecordsData from aiperf.common.models.dataset_models import ConversationMetadata, DatasetMetadata from aiperf.config import BenchmarkRun @@ -283,8 +284,8 @@ async def test_process_record_wraps_when_session_num_exceeds_dataset(self) -> No # session_num=1 wraps to index 0 (the only task, "algebra") await processor.process_record(_make_record_data(session_num=1)) - assert processor._task_total["algebra"] == 1 - assert processor._overall_total == 1 + assert processor._task_total[CreditPhase.PROFILING]["algebra"] == 1 + assert processor._overall_total[CreditPhase.PROFILING] == 1 async def test_process_record_wraps_to_correct_task(self) -> None: """With N problems, session_num=N+1 accumulates under the task at index 1.""" @@ -294,8 +295,8 @@ async def test_process_record_wraps_to_correct_task(self) -> None: # session_num=4 % 3 = index 1 → task="history" await processor.process_record(_make_record_data(session_num=4)) - assert processor._task_total["history"] == 1 - assert processor._task_total.get("algebra", 0) == 0 + assert processor._task_total[CreditPhase.PROFILING]["history"] == 1 + assert processor._task_total[CreditPhase.PROFILING].get("algebra", 0) == 0 async def test_process_record_last_valid_session_num_succeeds(self) -> None: processor = _make_accuracy_accumulator() @@ -303,9 +304,9 @@ async def test_process_record_last_valid_session_num_succeeds(self) -> None: await processor.process_record(_make_record_data(session_num=1, correct=1.0)) - assert processor._overall_total == 1 - assert processor._overall_correct == 1 - assert processor._task_correct["test_task"] == 1 + assert processor._overall_total[CreditPhase.PROFILING] == 1 + assert processor._overall_correct[CreditPhase.PROFILING] == 1 + assert processor._task_correct[CreditPhase.PROFILING]["test_task"] == 1 async def test_process_record_raises_if_not_configured(self) -> None: """process_record must raise if on_dataset_configured was never called.""" @@ -322,8 +323,8 @@ async def test_process_record_increments_overall_unparsed(self) -> None: _make_record_data(session_num=0, correct=1.0, unparsed=1.0) ) - assert processor._overall_unparsed == 1 - assert processor._overall_total == 1 + assert processor._overall_unparsed[CreditPhase.PROFILING] == 1 + assert processor._overall_total[CreditPhase.PROFILING] == 1 async def test_process_record_increments_task_unparsed(self) -> None: processor = _make_accuracy_accumulator() @@ -333,7 +334,7 @@ async def test_process_record_increments_task_unparsed(self) -> None: _make_record_data(session_num=0, correct=0.0, unparsed=1.0) ) - assert processor._task_unparsed["algebra"] == 1 + assert processor._task_unparsed[CreditPhase.PROFILING]["algebra"] == 1 async def test_process_record_does_not_increment_unparsed_when_conforming( self, @@ -345,8 +346,8 @@ async def test_process_record_does_not_increment_unparsed_when_conforming( _make_record_data(session_num=0, correct=1.0, unparsed=0.0) ) - assert processor._overall_unparsed == 0 - assert processor._task_unparsed.get("algebra", 0) == 0 + assert processor._overall_unparsed[CreditPhase.PROFILING] == 0 + assert processor._task_unparsed[CreditPhase.PROFILING].get("algebra", 0) == 0 async def test_process_record_missing_unparsed_key_treated_as_conforming( self, @@ -361,4 +362,4 @@ async def test_process_record_missing_unparsed_key_treated_as_conforming( await processor.process_record(data) - assert processor._overall_unparsed == 0 + assert processor._overall_unparsed[CreditPhase.PROFILING] == 0 diff --git a/tests/unit/accuracy/test_accuracy_results_processor_summarize.py b/tests/unit/accuracy/test_accuracy_results_processor_summarize.py index f7e6049064..d44c68bab0 100644 --- a/tests/unit/accuracy/test_accuracy_results_processor_summarize.py +++ b/tests/unit/accuracy/test_accuracy_results_processor_summarize.py @@ -4,6 +4,8 @@ import pytest from aiperf.accuracy.accuracy_results_processor import AccuracyResultsProcessor +from aiperf.common.accumulator_protocols import ExportContext +from aiperf.common.enums import CreditPhase from aiperf.plugin.enums import AccuracyBenchmarkType, EndpointType from tests.unit.conftest import make_benchmark_run @@ -19,6 +21,29 @@ def _make_processor() -> AccuracyResultsProcessor: ) +def _seed( + processor: AccuracyResultsProcessor, + *, + phase: CreditPhase = CreditPhase.PROFILING, + overall_total: int = 0, + overall_correct: int = 0, + overall_unparsed: int = 0, + task_total: dict[str, int] | None = None, + task_correct: dict[str, int] | None = None, + task_unparsed: dict[str, int] | None = None, +) -> None: + """Seed the per-phase counters for a single phase.""" + processor._overall_total[phase] = overall_total + processor._overall_correct[phase] = overall_correct + processor._overall_unparsed[phase] = overall_unparsed + for task, count in (task_total or {}).items(): + processor._task_total[phase][task] = count + for task, count in (task_correct or {}).items(): + processor._task_correct[phase][task] = count + for task, count in (task_unparsed or {}).items(): + processor._task_unparsed[phase][task] = count + + @pytest.mark.asyncio class TestAccuracyResultsProcessorSummarize: async def test_empty_returns_no_results(self) -> None: @@ -28,8 +53,7 @@ async def test_empty_returns_no_results(self) -> None: async def test_overall_metric_values(self) -> None: processor = _make_processor() - processor._overall_total = 10 - processor._overall_correct = 7 + _seed(processor, overall_total=10, overall_correct=7) results = await processor.summarize() @@ -41,12 +65,13 @@ async def test_overall_metric_values(self) -> None: async def test_task_metrics_sorted_alphabetically(self) -> None: processor = _make_processor() - processor._overall_total = 4 - processor._overall_correct = 3 - processor._task_total["zebra"] = 2 - processor._task_total["algebra"] = 2 - processor._task_correct["zebra"] = 1 - processor._task_correct["algebra"] = 2 + _seed( + processor, + overall_total=4, + overall_correct=3, + task_total={"zebra": 2, "algebra": 2}, + task_correct={"zebra": 1, "algebra": 2}, + ) results = await processor.summarize() task_results = [r for r in results if r.tag.startswith("accuracy.task.")] @@ -56,10 +81,13 @@ async def test_task_metrics_sorted_alphabetically(self) -> None: async def test_task_metric_accuracy_calculation(self) -> None: processor = _make_processor() - processor._overall_total = 5 - processor._overall_correct = 3 - processor._task_total["math"] = 5 - processor._task_correct["math"] = 3 + _seed( + processor, + overall_total=5, + overall_correct=3, + task_total={"math": 5}, + task_correct={"math": 3}, + ) results = await processor.summarize() @@ -71,8 +99,7 @@ async def test_task_metric_accuracy_calculation(self) -> None: async def test_overall_not_emitted_when_no_results_processed(self) -> None: processor = _make_processor() - processor._task_total["math"] = 3 - processor._task_correct["math"] = 2 + _seed(processor, task_total={"math": 3}, task_correct={"math": 2}) results = await processor.summarize() @@ -83,11 +110,13 @@ async def test_overall_not_emitted_when_no_results_processed(self) -> None: async def test_multiple_tasks_each_get_own_metric(self) -> None: processor = _make_processor() - processor._overall_total = 6 - processor._overall_correct = 4 - for task in ("history", "biology", "physics"): - processor._task_total[task] = 2 - processor._task_correct[task] = 1 + _seed( + processor, + overall_total=6, + overall_correct=4, + task_total={"history": 2, "biology": 2, "physics": 2}, + task_correct={"history": 1, "biology": 1, "physics": 1}, + ) results = await processor.summarize() task_tags = {r.tag for r in results if r.tag.startswith("accuracy.task.")} @@ -100,9 +129,7 @@ async def test_multiple_tasks_each_get_own_metric(self) -> None: async def test_unparsed_overall_emitted_when_records_processed(self) -> None: processor = _make_processor() - processor._overall_total = 10 - processor._overall_correct = 7 - processor._overall_unparsed = 3 + _seed(processor, overall_total=10, overall_correct=7, overall_unparsed=3) results = await processor.summarize() @@ -113,11 +140,14 @@ async def test_unparsed_overall_emitted_when_records_processed(self) -> None: async def test_unparsed_per_task_emitted(self) -> None: processor = _make_processor() - processor._overall_total = 5 - processor._overall_correct = 3 - processor._task_total["math"] = 5 - processor._task_correct["math"] = 3 - processor._task_unparsed["math"] = 2 + _seed( + processor, + overall_total=5, + overall_correct=3, + task_total={"math": 5}, + task_correct={"math": 3}, + task_unparsed={"math": 2}, + ) results = await processor.summarize() @@ -130,13 +160,44 @@ async def test_unparsed_per_task_emitted(self) -> None: async def test_unparsed_zero_when_all_conforming(self) -> None: processor = _make_processor() - processor._overall_total = 5 - processor._overall_correct = 5 - processor._task_total["math"] = 5 - processor._task_correct["math"] = 5 + _seed( + processor, + overall_total=5, + overall_correct=5, + task_total={"math": 5}, + task_correct={"math": 5}, + ) results = await processor.summarize() unparsed = next(r for r in results if r.tag == "accuracy.unparsed") assert unparsed.sum == 0 assert unparsed.current == pytest.approx(0.0) + + async def test_export_results_scopes_to_phase(self) -> None: + """export_results(ctx) must isolate the requested phase so warmup grades + do not leak into the profiling accuracy summary (and vice versa).""" + processor = _make_processor() + # One correct warmup record, one incorrect profiling record. + _seed(processor, phase=CreditPhase.WARMUP, overall_total=1, overall_correct=1) + _seed( + processor, phase=CreditPhase.PROFILING, overall_total=1, overall_correct=0 + ) + + profiling = await processor.export_results( + ExportContext(phase=CreditPhase.PROFILING) + ) + overall = next(r for r in profiling if r.tag == "accuracy.overall") + assert overall.count == 1 + assert overall.current == pytest.approx(0.0) + + warmup = await processor.export_results(ExportContext(phase=CreditPhase.WARMUP)) + overall = next(r for r in warmup if r.tag == "accuracy.overall") + assert overall.count == 1 + assert overall.current == pytest.approx(1.0) + + # summarize() (phase-agnostic) still sees both records combined. + combined = await processor.summarize() + overall = next(r for r in combined if r.tag == "accuracy.overall") + assert overall.count == 2 + assert overall.current == pytest.approx(0.5) From f219a0e54f77c335e7b71b3f8e4399fc5ed548b3 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 16:54:36 -0700 Subject: [PATCH 03/39] fix(records): surface metric-handler failures in the phase error summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_on_metric_records` discarded the exceptions returned by `_dispatch_record()` and immediately marked the record processed, so a failed metric accumulator or stream exporter produced incomplete metrics while `completed` advanced and the phase error summary omitted the failure. Record each dispatch error via `_error_tracker.increment_error_count_for_phase(...)` — matching the telemetry, server-metrics, and network-latency paths — so handler failures are no longer silently dropped. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/records/records_manager.py | 13 +++-- tests/unit/records/test_records_manager.py | 55 ++++++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 17a164e6e0..9dfbda34bd 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -532,14 +532,21 @@ async def _on_metric_records(self, message: MetricRecordsMessage) -> None: self._maybe_hint_missing_cache_reporting(record_data) - await self._dispatch_record(record_data) + phase = record_data.metadata.benchmark_phase + dispatch_errors = await self._dispatch_record(record_data) self._records_tracker.update_from_record_data(record_data) if record_data.error: self._error_tracker.increment_error_count_for_phase( - record_data.metadata.benchmark_phase, record_data.error + phase, record_data.error + ) + # A metric accumulator/exporter that failed to ingest this record yields + # incomplete metrics; surface it in the phase error summary rather than + # marking the record cleanly processed and silently dropping the failure. + for error in dispatch_errors: + self._error_tracker.increment_error_count_for_phase( + phase, ErrorDetails.from_exception(error) ) - phase = record_data.metadata.benchmark_phase if ( phase in self._complete_credit_phases and self._records_tracker.check_and_set_all_records_received_for_phase( diff --git a/tests/unit/records/test_records_manager.py b/tests/unit/records/test_records_manager.py index 4873ae1a91..4b2684a928 100644 --- a/tests/unit/records/test_records_manager.py +++ b/tests/unit/records/test_records_manager.py @@ -39,6 +39,7 @@ from aiperf.metrics.accumulator_models import AccumulatorMetricsSummary from aiperf.metrics.cache_reporting_hint import CACHE_REPORTING_HINT from aiperf.plugin.enums import AccumulatorType, TimingMode +from aiperf.records.error_tracker import ErrorTracker from aiperf.records.records_manager import ErrorTrackingState, RecordsManager from aiperf.records.records_tracker import RecordsTracker from aiperf.timing.config import CreditPhaseConfig @@ -144,6 +145,60 @@ async def test_on_telemetry_records_invalid_tracks_error(self) -> None: manager._dispatch_record.assert_not_awaited() +class TestRecordsManagerMetricRecordDispatchErrors: + """Metric-handler failures must surface in the phase error summary rather + than being silently dropped while the record is marked processed.""" + + def _make_manager(self) -> RecordsManager: + manager = RecordsManager.__new__(RecordsManager) + manager.debug = MagicMock() + manager.error = MagicMock() + manager.trace = MagicMock() + manager.is_enabled_for = MagicMock(return_value=False) + manager._dataset_configured_event = asyncio.Event() + manager._dataset_configured_event.set() + manager._maybe_hint_missing_cache_reporting = MagicMock() + manager._records_tracker = MagicMock() + manager._records_tracker.check_and_set_all_records_received_for_phase.return_value = False + manager._error_tracker = ErrorTracker() + manager._complete_credit_phases = set() + return manager + + @pytest.mark.asyncio + async def test_metric_dispatch_error_recorded_in_phase_error_summary(self) -> None: + manager = self._make_manager() + dispatch_error = RuntimeError("metric accumulator failed") + manager._dispatch_record = AsyncMock(return_value=[dispatch_error]) + + message = MagicMock() + message.to_data.return_value = create_metric_record_data(1_000, 2_000) + + await manager._on_metric_records(message) + + # Record is still counted, but the handler failure is not swallowed. + manager._records_tracker.update_from_record_data.assert_called_once() + summary = manager._error_tracker.get_error_summary_for_phase( + CreditPhase.PROFILING + ) + tracked = ErrorDetails.from_exception(dispatch_error) + assert any(e.error_details == tracked for e in summary) + + @pytest.mark.asyncio + async def test_successful_metric_dispatch_records_no_phase_error(self) -> None: + manager = self._make_manager() + manager._dispatch_record = AsyncMock(return_value=[]) + + message = MagicMock() + message.to_data.return_value = create_metric_record_data(1_000, 2_000) + + await manager._on_metric_records(message) + + assert ( + manager._error_tracker.get_error_summary_for_phase(CreditPhase.PROFILING) + == [] + ) + + class TestRecordsManagerTimeslice: """ProfileResults stores accumulator-backed timeslices.""" From 07f6d05048b8e4a96b61842ad8d9cd2871a0c6c7 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 16:57:07 -0700 Subject: [PATCH 04/39] docs/test: address CodeRabbit review nitpicks - plugin-system.md: correct the plugin-category total 33 -> 32 (categories.yaml has 32 real categories after the results_processor -> accumulator/stream_exporter taxonomy change). - network_adjusted_metrics.py: document the zero-clamping caveat. The per-record transform is max(latency - rtt, 0), so the "shift preserves percentiles/mean and leaves std unchanged" invariant only holds for records not clamped at zero. - test_telemetry_integration.py: annotate failing_process_telemetry_record. - test_record_models.py: name the four ProfileResults tests per convention and add -> None return annotations. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/plugins/plugin-system.md | 2 +- src/aiperf/metrics/types/network_adjusted_metrics.py | 9 ++++++--- tests/unit/common/models/test_record_models.py | 8 ++++---- tests/unit/gpu_telemetry/test_telemetry_integration.py | 3 ++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/plugins/plugin-system.md b/docs/plugins/plugin-system.md index 58c0c22864..d8151ee623 100644 --- a/docs/plugins/plugin-system.md +++ b/docs/plugins/plugin-system.md @@ -100,7 +100,7 @@ for entry, cls in plugins.iter_all(PluginType.ENDPOINT): ## Plugin Categories -AIPerf supports 33 plugin categories organized by function, including `api_router` and `public_dataset_loader`: +AIPerf supports 32 plugin categories organized by function, including `api_router` and `public_dataset_loader`: ### Timing Categories diff --git a/src/aiperf/metrics/types/network_adjusted_metrics.py b/src/aiperf/metrics/types/network_adjusted_metrics.py index 70847d5708..4cd7a38f03 100644 --- a/src/aiperf/metrics/types/network_adjusted_metrics.py +++ b/src/aiperf/metrics/types/network_adjusted_metrics.py @@ -9,9 +9,12 @@ These metrics are NOT computed per-record. The RTT is a run-level scalar that is only known at the end of the run, so MetricsAccumulator injects the shifted distributions -after aggregation (subtracting a constant shifts every percentile and the mean by that -constant and leaves the standard deviation unchanged). ``_derive_value`` therefore -always defers. +after aggregation. For records where the RTT does not exceed the source latency the +adjustment is a pure shift (subtracting a constant shifts every percentile and the mean +by that constant and leaves the standard deviation unchanged). The per-record transform +is ``max(latency - rtt, 0)``, though, so any record whose latency is below the RTT is +clamped at zero; when that happens the shift is no longer uniform and the percentile / +mean / std relationships above no longer hold exactly. ``_derive_value`` always defers. Inter-token / inter-chunk latencies are intentionally NOT adjusted: the same RTT cancels in ``(request_latency - ttft)``, so those metrics are already network-invariant. diff --git a/tests/unit/common/models/test_record_models.py b/tests/unit/common/models/test_record_models.py index 1e97641e82..5878553802 100644 --- a/tests/unit/common/models/test_record_models.py +++ b/tests/unit/common/models/test_record_models.py @@ -17,7 +17,7 @@ class TestProfileResults: """Test cases for ProfileResults model.""" - def test_profile_results_with_timeslices(self): + def test_profile_results_timeslices_preserves_metric_lookup(self) -> None: """Test ProfileResults stores accumulator-backed timeslices.""" metric_result = MetricResult( tag="request_latency", @@ -58,7 +58,7 @@ def test_profile_results_with_timeslices(self): is metric_result ) - def test_profile_results_without_timeslices(self): + def test_profile_results_no_timeslices_defaults_to_none(self) -> None: """Test ProfileResults works without timeslice results.""" metric_result = MetricResult( tag="request_latency", @@ -77,7 +77,7 @@ def test_profile_results_without_timeslices(self): assert profile_results.timeslices is None - def test_profile_results_with_empty_timeslices(self): + def test_profile_results_empty_timeslices_preserves_empty_list(self) -> None: """Test ProfileResults with empty timeslice list.""" metric_result = MetricResult( tag="request_latency", @@ -97,7 +97,7 @@ def test_profile_results_with_empty_timeslices(self): assert profile_results.timeslices == [] - def test_profile_results_with_multiple_timeslices_and_metrics(self): + def test_profile_results_multiple_timeslices_maps_metrics_by_tag(self) -> None: """Test ProfileResults with multiple timeslices containing multiple metrics.""" latency_result = MetricResult( tag="request_latency", diff --git a/tests/unit/gpu_telemetry/test_telemetry_integration.py b/tests/unit/gpu_telemetry/test_telemetry_integration.py index 4bc85c0e75..1130403f4e 100644 --- a/tests/unit/gpu_telemetry/test_telemetry_integration.py +++ b/tests/unit/gpu_telemetry/test_telemetry_integration.py @@ -13,6 +13,7 @@ import aiohttp import pytest +from aiperf.common.models import TelemetryRecord from aiperf.config.flags.cli_config import CLIConfig from aiperf.gpu_telemetry.accumulator import GPUTelemetryAccumulator from aiperf.gpu_telemetry.dcgm_collector import DCGMTelemetryCollector @@ -269,7 +270,7 @@ async def test_callback_pipeline_error_handling( # Mock the process_telemetry_record method to raise an exception original_process = faulty_processor.process_telemetry_record - async def failing_process_telemetry_record(record): + async def failing_process_telemetry_record(record: TelemetryRecord) -> None: if record.gpu_index == 0: # Fail on first GPU only raise ValueError("Simulated processing error") return await original_process(record) From 68e2e27a336355b93b3803d37448d8ada18d4b96 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 17:50:09 -0700 Subject: [PATCH 05/39] fix(metrics): port replay_sched_lag to a columnar analyzer; delete legacy MetricArray The replay send-lag family (replay_sched_lag_p50/p90/p99 + replay_sched_degraded) came in from origin/main (baseten_trace #1129) but was never wired into the columnar accumulator: its derive read a MetricArray-shaped `.data` off scalar_dict, which the accumulator stores as a bare scalar sum, so every summarize raised `AttributeError("'float' object has no attribute 'data'")` and the whole family silently vanished from fixed-schedule exports. Port it the accumulator-native way, mirroring network_adjusted and derived_latency: - Metric classes in types/replay_sched_lag_metrics.py now defer (`_derive_value` raises NoMetricValue). - New metrics/replay_sched_lag_analyzer.py owns the columnar computation: `inject_replay_sched_lag_metrics(store, results, mask, warn_degraded=...)` computes the anchored-percentile distribution + degraded flag directly from the `replay_send_schedule_offset` column and injects the MetricResults. Run-scoped (anchored at the run-global least-late request), injected once over the masked column, never per timeslice. Sits beside network_adjusted_analyzer.py. - MetricsAccumulator calls it in summarize() and owns the once-per-run "schedule degraded" warning latch. Delete the retired MetricArray (dead since the ColumnStore refactor; only docstrings and test oracles still referenced it): - Remove the class + exports; fix list_metric_aggregation docstrings. - Aggregation oracle uses metric_result_from_array (numpy-exact) instead. - DerivedSumMetric total_* tests feed a scalar sum (what the accumulator actually stores) instead of a MetricArray. Correctness: a golden-parity test pins the injection to the exact legacy `MetricResultsProcessor` formula across seeds + edge cases (anchoring, uniform lag, degraded threshold, masking); a live tuned-mock trace reproduced the legacy formula on its 50 real offsets with 0.0 difference on all four metrics. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/metrics/__init__.py | 2 - src/aiperf/metrics/accumulator.py | 30 +++ src/aiperf/metrics/list_metric_aggregation.py | 4 +- src/aiperf/metrics/metric_dicts.py | 77 ------ .../metrics/replay_sched_lag_analyzer.py | 107 ++++++++ .../metrics/types/replay_sched_lag_metrics.py | 86 ++---- tests/unit/metrics/conftest.py | 10 +- .../test_input_sequence_length_metric.py | 5 +- .../metrics/test_list_metric_aggregation.py | 18 +- tests/unit/metrics/test_metric_array.py | 137 ---------- .../test_output_sequence_length_metric.py | 3 +- tests/unit/metrics/test_output_token_count.py | 3 +- .../metrics/test_reasoning_token_count.py | 3 +- .../metrics/test_replay_sched_lag_metrics.py | 246 +++++++++++------- 14 files changed, 328 insertions(+), 403 deletions(-) create mode 100644 src/aiperf/metrics/replay_sched_lag_analyzer.py delete mode 100644 tests/unit/metrics/test_metric_array.py diff --git a/src/aiperf/metrics/__init__.py b/src/aiperf/metrics/__init__.py index cfb9570a28..a2cccd737f 100644 --- a/src/aiperf/metrics/__init__.py +++ b/src/aiperf/metrics/__init__.py @@ -10,7 +10,6 @@ from aiperf.metrics.derived_sum_metric import DerivedSumMetric, RecordMetricT from aiperf.metrics.metric_dicts import ( BaseMetricDict, - MetricArray, MetricDictValueTypeVarT, MetricRecordDict, MetricResultsDict, @@ -25,7 +24,6 @@ "BaseMetricDict", "BaseRecordMetric", "DerivedSumMetric", - "MetricArray", "MetricDictValueTypeVarT", "MetricRecordDict", "MetricRegistry", diff --git a/src/aiperf/metrics/accumulator.py b/src/aiperf/metrics/accumulator.py index 67dc90f443..92fb7befc0 100644 --- a/src/aiperf/metrics/accumulator.py +++ b/src/aiperf/metrics/accumulator.py @@ -38,6 +38,10 @@ compute_network_adjusted_arrays, inject_network_adjusted_from_arrays, ) +from aiperf.metrics.replay_sched_lag_analyzer import inject_replay_sched_lag_metrics +from aiperf.metrics.types.replay_sched_lag_metrics import ( + REPLAY_SCHED_DEGRADED_THRESHOLD_MS, +) from aiperf.post_processors.base_metrics_processor import BaseMetricsProcessor if TYPE_CHECKING: @@ -96,6 +100,10 @@ def __init__( # summarize() when network latency calibration is active. None = no-op. self._network_rtt_ns: float | None = None + # One-shot latch for the run-level "replay schedule degraded" warning + # emitted by inject_replay_sched_lag_metrics (fires at most once per run). + self._replay_degraded_warned: bool = False + # Derive functions for DERIVED metrics # _setup_metrics includes transitive dependencies (RECORD/AGGREGATE), # so filter to only metrics that actually have derive_value. @@ -398,6 +406,19 @@ def _collect_one_list_column( # uniformly via the scalar_dict. scalar_dict[tag] = float(backend.sum) + def _warn_replay_degraded(self, p50: float, p90: float, p99: float) -> None: + """Emit the run-level replay-schedule-degraded warning at most once.""" + if self._replay_degraded_warned: + return + self._replay_degraded_warned = True + self.warning( + f"Replay schedule degraded: anchored send lag p50={p50:.0f} ms, " + f"p90={p90:.0f} ms, p99={p99:.0f} ms exceeds " + f"{REPLAY_SCHED_DEGRADED_THRESHOLD_MS:.0f} ms. Request timing no longer " + f"tracks the recorded schedule; consider lowering replay_speedup or the " + f"offered load." + ) + def _resolve_derived_metrics( self, scalar_dict: MetricResultsDict, *, is_timeslice: bool = False ) -> None: @@ -558,6 +579,15 @@ def _summarize_for_export_context( self._network_rtt_ns, mask=mask, ) + # Run-scoped replay send-lag family (fixed-schedule only): anchored at + # the run-global least-late request, so computed once over the masked + # offset column, never per timeslice. + inject_replay_sched_lag_metrics( + self._column_store, + overall_results, + mask=mask, + warn_degraded=self._warn_replay_degraded, + ) overall_results = self._filter_hidden_metrics(overall_results) self.debug(lambda: f"Summarized {len(overall_results)} metric results") diff --git a/src/aiperf/metrics/list_metric_aggregation.py b/src/aiperf/metrics/list_metric_aggregation.py index ca1bef1aed..66d9ae4d2d 100644 --- a/src/aiperf/metrics/list_metric_aggregation.py +++ b/src/aiperf/metrics/list_metric_aggregation.py @@ -68,7 +68,7 @@ def __init__(self) -> None: def sum(self) -> float: """Exact running sum of all samples — for the :class:`MetricSeriesProtocol` so derived-sum metrics can compute - uniformly across this and :class:`MetricArray`.""" + uniformly across every MetricAggregator implementation.""" return self._sum def __len__(self) -> int: @@ -131,7 +131,7 @@ def add_for_record(self, idx: int, values: list[float]) -> None: # noqa: ARG002 def to_result(self, tag: MetricTagT, header: str, unit: str) -> MetricResult: """Return a :class:`MetricResult` with the same field set as - ``MetricArray.to_result``. Percentiles come from the t-digest; + the exact numpy-percentile reference. Percentiles come from the t-digest; every other stat is exact.""" if self._count == 0: return MetricResult(tag=tag, header=header, unit=unit, count=0) diff --git a/src/aiperf/metrics/metric_dicts.py b/src/aiperf/metrics/metric_dicts.py index d693ba6a6c..a78c0fa161 100644 --- a/src/aiperf/metrics/metric_dicts.py +++ b/src/aiperf/metrics/metric_dicts.py @@ -13,11 +13,8 @@ MetricType, MetricUnitT, MetricValueTypeT, - MetricValueTypeVarT, ) -from aiperf.common.environment import Environment from aiperf.common.exceptions import MetricTypeError, MetricUnitError, NoMetricValue -from aiperf.common.growable_array import GrowableArray from aiperf.common.models.record_models import MetricResult, MetricValue from aiperf.common.types import MetricTagT @@ -29,7 +26,6 @@ __all__ = [ "BaseMetricDict", "MetricAggregator", - "MetricArray", "MetricDictValueTypeVarT", "MetricRecordDict", "MetricResultsDict", @@ -251,76 +247,3 @@ def get_converted_or_raise( f"Cannot convert a record metric to a different unit: {metric.tag}" ) return super().get_converted_or_raise(metric, other_unit) - - -class MetricArray(Generic[MetricValueTypeVarT]): - """NumPy backed array for metric data. - - This is used to store the values of a metric over time. - Uses GrowableArray internally for efficient storage with automatic growth. - """ - - def __init__( - self, initial_capacity: int = Environment.METRICS.ARRAY_INITIAL_CAPACITY - ): - """Initialize the array with the given initial capacity.""" - self._array = GrowableArray( - initial_capacity=initial_capacity, - dtype=np.float64, - track_sum=True, - ) - - def extend(self, values: list[MetricValueTypeVarT]) -> None: - """Extend the array with a list of values.""" - self._array.extend(np.asarray(values, dtype=np.float64)) - - def append(self, value: MetricValueTypeVarT) -> None: - """Append a value to the array.""" - self._array.append(value) - - @property - def sum(self) -> MetricValueTypeVarT: - """Get the sum of the array.""" - return self._array.sum # type: ignore - - @property - def data(self) -> np.ndarray: - """Return view of actual data.""" - return self._array.data - - @property - def capacity(self) -> int: - """Return current capacity.""" - return self._array.capacity - - def __len__(self) -> int: - """Return number of elements.""" - return len(self._array) - - def to_result(self, tag: MetricTagT, header: str, unit: str) -> MetricResult: - """Compute metric stats with zero-copy""" - - arr = self.data - p1, p5, p10, p25, p50, p75, p90, p95, p99 = np.percentile( - arr, [1, 5, 10, 25, 50, 75, 90, 95, 99] - ) - return MetricResult( - tag=tag, - header=header, - unit=unit, - min=np.min(arr), - max=np.max(arr), - avg=float(np.mean(arr)), - sum=float(self.sum), - std=float(np.std(arr)), - p1=p1, - p5=p5, - p10=p10, - p25=p25, - p50=p50, - p75=p75, - p90=p90, - p95=p95, - p99=p99, - count=len(self._array), - ) diff --git a/src/aiperf/metrics/replay_sched_lag_analyzer.py b/src/aiperf/metrics/replay_sched_lag_analyzer.py new file mode 100644 index 0000000000..dd2fe4a80a --- /dev/null +++ b/src/aiperf/metrics/replay_sched_lag_analyzer.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Baseten.co. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Replay send-lag analyzer: compute the ``replay_sched_lag_*`` family from the +column store at summarize-time. + +The metrics themselves are defined in +:mod:`aiperf.metrics.types.replay_sched_lag_metrics` and defer their derivation; +this module owns the columnar computation and injection, mirroring +:mod:`aiperf.metrics.network_adjusted_analyzer` and +:mod:`aiperf.metrics.derived_latency`. + +The family is a distribution over the run-global ``replay_send_schedule_offset`` +column anchored at the least-late request, so it is computed once over the full +(masked) column and never per timeslice. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +from numpy.typing import NDArray + +from aiperf.common.constants import NANOS_PER_MILLIS +from aiperf.common.models import MetricResult +from aiperf.metrics.types.replay_sched_lag_metrics import ( + REPLAY_SCHED_DEGRADED_THRESHOLD_MS, + ReplaySchedDegradedMetric, + ReplaySchedLagP50Metric, + ReplaySchedLagP90Metric, + ReplaySchedLagP99Metric, + ReplaySchedLagPercentileBase, + ReplaySendScheduleOffsetMetric, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + from aiperf.metrics.column_store import ColumnStore + + +# The ``percentile`` each class declares is the single source of truth for both +# its identity and the value the analyzer computes. +_PERCENTILE_METRICS: tuple[type[ReplaySchedLagPercentileBase], ...] = ( + ReplaySchedLagP50Metric, + ReplaySchedLagP90Metric, + ReplaySchedLagP99Metric, +) + + +def inject_replay_sched_lag_metrics( + store: ColumnStore, + results: dict[str, MetricResult], + mask: NDArray[np.bool_] | None = None, + *, + warn_degraded: Callable[[float, float, float], None] | None = None, +) -> None: + """Inject the anchored replay send-lag percentiles + degraded flag. + + Run-scoped: the family anchors at the run-global least-late request, so it is + computed once over the full (masked) ``replay_send_schedule_offset`` column + and never per timeslice. No-op when the column is absent (non-fixed-schedule + runs) or holds no absolutely-scheduled offsets. ``warn_degraded``, if given, + is called with ``(p50, p90, p99)`` once when the run is flagged degraded so + the caller can emit a single run-level warning. Pure side effect on + ``results``. + """ + if ReplaySendScheduleOffsetMetric.tag not in store.numeric_tags(): + return + offsets = store.numeric(ReplaySendScheduleOffsetMetric.tag) + if mask is not None: + offsets = offsets[mask] + offsets = offsets[~np.isnan(offsets)] + if offsets.size == 0: + return + + anchored_ms = (offsets - offsets.min()) / NANOS_PER_MILLIS + # Each percentile is a single run-level derived scalar (count=1), matching + # the legacy derive path and the sibling network_rtt injection. + lag: dict[str, float] = {} + for cls in _PERCENTILE_METRICS: + value = float(np.percentile(anchored_ms, cls.percentile)) + lag[cls.tag] = value + results[cls.tag] = MetricResult( + tag=cls.tag, + header=cls.header, + unit=str(cls.unit), + avg=value, + count=1, + console_group=cls.console_group, + ) + + degraded = lag[ReplaySchedLagP99Metric.tag] > REPLAY_SCHED_DEGRADED_THRESHOLD_MS + results[ReplaySchedDegradedMetric.tag] = MetricResult( + tag=ReplaySchedDegradedMetric.tag, + header=ReplaySchedDegradedMetric.header, + unit=str(ReplaySchedDegradedMetric.unit), + avg=float(int(degraded)), + count=1, + console_group=ReplaySchedDegradedMetric.console_group, + ) + if degraded and warn_degraded is not None: + warn_degraded( + lag[ReplaySchedLagP50Metric.tag], + lag[ReplaySchedLagP90Metric.tag], + lag[ReplaySchedLagP99Metric.tag], + ) diff --git a/src/aiperf/metrics/types/replay_sched_lag_metrics.py b/src/aiperf/metrics/types/replay_sched_lag_metrics.py index 3408ff48ad..d3df39abcc 100644 --- a/src/aiperf/metrics/types/replay_sched_lag_metrics.py +++ b/src/aiperf/metrics/types/replay_sched_lag_metrics.py @@ -38,9 +38,9 @@ cumulative schedule drift these metrics exist to expose. """ -from typing import ClassVar +from __future__ import annotations -import numpy as np +from typing import ClassVar from aiperf.common.constants import NANOS_PER_MILLIS from aiperf.common.enums import ( @@ -50,14 +50,11 @@ MetricTimeUnit, ) from aiperf.common.exceptions import NoMetricValue -from aiperf.common.logging import AIPerfLogger from aiperf.common.models import ParsedResponseRecord from aiperf.metrics.base_derived_metric import BaseDerivedMetric from aiperf.metrics.base_record_metric import BaseRecordMetric from aiperf.metrics.metric_dicts import MetricRecordDict, MetricResultsDict -_logger = AIPerfLogger(__name__) - REPLAY_SCHED_DEGRADED_THRESHOLD_MS: float = 500.0 """Anchored send-lag p99 (ms) above which a replay run is flagged degraded. @@ -112,34 +109,26 @@ def _parse_record( return record.timestamp_ns - int(intended_ms * NANOS_PER_MILLIS) -_lag_cache: tuple[object, int, np.ndarray] | None = None +class _ReplaySchedLagDeferMixin: + """Deferred-derivation behavior for the injected replay send-lag metrics. + Not a metric itself (does not subclass BaseMetric) so it is never registered. + The whole family is a distribution over the run-global + ``replay_send_schedule_offset`` column, which the scalar summarize path does + not expose, so the derive defers and :func:`inject_replay_sched_lag_metrics` + fills the values post-aggregation from the column store -- mirroring how + ``network_adjusted_*`` and the derived-latency family are injected. + """ -def _anchored_lag_ms(metric_results: MetricResultsDict) -> np.ndarray: - """Return per-request send lag in ms, anchored at the least-late request. + def _derive_value(self, metric_results: MetricResultsDict): + raise NoMetricValue( + f"{self.tag} is injected post-aggregation from the " # type: ignore[attr-defined] + "replay_send_schedule_offset column" + ) - The p50/p90/p99 metrics derive back-to-back from the same offsets, so the - last computation is cached, keyed on array identity and length (offset - arrays are append-only). - Raises: - NoMetricValue: If no absolutely-scheduled request produced an offset. - """ - global _lag_cache - values = metric_results.get_or_raise(ReplaySendScheduleOffsetMetric) - size = len(values.data) - if size == 0: - raise NoMetricValue("No absolutely-scheduled requests were recorded.") - if _lag_cache is not None and _lag_cache[0] is values and _lag_cache[1] == size: - return _lag_cache[2] - data = np.asarray(values.data, dtype=np.float64) - anchored = (data - data.min()) / NANOS_PER_MILLIS - _lag_cache = (values, size, anchored) - return anchored - - -class _ReplaySchedLagPercentileBase(BaseDerivedMetric[float]): - """Shared derive logic for the anchored send-lag percentile metrics.""" +class ReplaySchedLagPercentileBase(_ReplaySchedLagDeferMixin, BaseDerivedMetric[float]): + """Shared metadata for the anchored send-lag percentile metrics.""" __is_abstract__ = True percentile: float @@ -150,11 +139,8 @@ class _ReplaySchedLagPercentileBase(BaseDerivedMetric[float]): timeslice_derivable = False required_metrics: ClassVar[set[str]] = {ReplaySendScheduleOffsetMetric.tag} - def _derive_value(self, metric_results: MetricResultsDict) -> float: - return float(np.percentile(_anchored_lag_ms(metric_results), self.percentile)) - -class ReplaySchedLagP50Metric(_ReplaySchedLagPercentileBase): +class ReplaySchedLagP50Metric(ReplaySchedLagPercentileBase): """Median anchored send lag of a fixed-schedule replay run, in ms. Formula: @@ -168,7 +154,7 @@ class ReplaySchedLagP50Metric(_ReplaySchedLagPercentileBase): short_header = "Sched Lag p50" -class ReplaySchedLagP90Metric(_ReplaySchedLagPercentileBase): +class ReplaySchedLagP90Metric(ReplaySchedLagPercentileBase): """90th-percentile anchored send lag of a fixed-schedule replay run, in ms. Formula: @@ -182,7 +168,7 @@ class ReplaySchedLagP90Metric(_ReplaySchedLagPercentileBase): short_header = "Sched Lag p90" -class ReplaySchedLagP99Metric(_ReplaySchedLagPercentileBase): +class ReplaySchedLagP99Metric(ReplaySchedLagPercentileBase): """99th-percentile anchored send lag of a fixed-schedule replay run, in ms. Formula: @@ -196,15 +182,15 @@ class ReplaySchedLagP99Metric(_ReplaySchedLagPercentileBase): short_header = "Sched Lag p99" -class ReplaySchedDegradedMetric(BaseDerivedMetric[int]): +class ReplaySchedDegradedMetric(_ReplaySchedLagDeferMixin, BaseDerivedMetric[int]): """Boolean (0/1) signal that the replay could not keep up with the offered schedule: anchored send-lag p99 exceeded :data:`REPLAY_SCHED_DEGRADED_THRESHOLD_MS`. - Logs a one-line warning (at most once per run) with the lag percentiles - when degraded, so runs whose timing fidelity is compromised are called out - even though these metrics are hidden from the console table. The metric - value itself stays continuous: it is re-derived on every summarize tick. + Injected by :func:`inject_replay_sched_lag_metrics`, which also emits a + one-line warning (at most once per run) with the lag percentiles when + degraded, so runs whose timing fidelity is compromised are called out even + though these metrics are hidden from the console table. Formula: 1 if p99(offset - min(offset)) > REPLAY_SCHED_DEGRADED_THRESHOLD_MS else 0 @@ -222,23 +208,3 @@ class ReplaySchedDegradedMetric(BaseDerivedMetric[int]): ReplaySchedLagP90Metric.tag, ReplaySchedLagP99Metric.tag, } - - def __init__(self) -> None: - super().__init__() - self._warned = False - - def _derive_value(self, metric_results: MetricResultsDict) -> int: - p50 = metric_results.get_or_raise(ReplaySchedLagP50Metric) - p90 = metric_results.get_or_raise(ReplaySchedLagP90Metric) - p99 = metric_results.get_or_raise(ReplaySchedLagP99Metric) - degraded = p99 > REPLAY_SCHED_DEGRADED_THRESHOLD_MS - if degraded and not self._warned: - self._warned = True - _logger.warning( - f"Replay schedule degraded: anchored send lag p50={p50:.0f} ms, " - f"p90={p90:.0f} ms, p99={p99:.0f} ms exceeds " - f"{REPLAY_SCHED_DEGRADED_THRESHOLD_MS:.0f} ms. Request timing no " - f"longer tracks the recorded schedule; consider lowering " - f"replay_speedup or the offered load." - ) - return int(degraded) diff --git a/tests/unit/metrics/conftest.py b/tests/unit/metrics/conftest.py index 5d51fa700e..3bd0015611 100644 --- a/tests/unit/metrics/conftest.py +++ b/tests/unit/metrics/conftest.py @@ -22,7 +22,7 @@ ) from aiperf.common.models.record_models import TextResponseData, TokenCounts from aiperf.common.types import MetricTagT -from aiperf.metrics.metric_dicts import MetricArray, MetricRecordDict, MetricResultsDict +from aiperf.metrics.metric_dicts import MetricRecordDict, MetricResultsDict from aiperf.metrics.metric_registry import MetricRegistry from aiperf.plugin.enums import EndpointType @@ -148,11 +148,3 @@ def run_simple_metrics_pipeline( metric_results[metric.tag] = metric.derive_value(metric_results) return metric_results - - -def create_metric_array(values): - """Create a MetricArray with test values.""" - array = MetricArray() - if values: - array.extend(values) - return array diff --git a/tests/unit/metrics/test_input_sequence_length_metric.py b/tests/unit/metrics/test_input_sequence_length_metric.py index 097b26955b..260337f9f1 100644 --- a/tests/unit/metrics/test_input_sequence_length_metric.py +++ b/tests/unit/metrics/test_input_sequence_length_metric.py @@ -13,7 +13,6 @@ TotalInputSequenceLengthMetric, ) from tests.unit.metrics.conftest import ( - create_metric_array, create_record, run_simple_metrics_pipeline, ) @@ -69,7 +68,7 @@ def test_sum_calculation(self, values, expected_sum): """Test that TotalInputSequenceLengthMetric correctly sums all input tokens""" metric = TotalInputSequenceLengthMetric() metric_results = MetricResultsDict() - metric_results[InputSequenceLengthMetric.tag] = create_metric_array(values) + metric_results[InputSequenceLengthMetric.tag] = sum(values) result = metric.derive_value(metric_results) assert result == expected_sum @@ -132,7 +131,7 @@ def test_sum_calculation(self, values, expected_sum): """Test that TotalErrorInputSequenceLengthMetric correctly sums error input tokens""" metric = TotalErrorInputSequenceLengthMetric() metric_results = MetricResultsDict() - metric_results[ErrorInputSequenceLengthMetric.tag] = create_metric_array(values) + metric_results[ErrorInputSequenceLengthMetric.tag] = sum(values) result = metric.derive_value(metric_results) assert result == expected_sum diff --git a/tests/unit/metrics/test_list_metric_aggregation.py b/tests/unit/metrics/test_list_metric_aggregation.py index 1ae83b888b..ea591b2a71 100644 --- a/tests/unit/metrics/test_list_metric_aggregation.py +++ b/tests/unit/metrics/test_list_metric_aggregation.py @@ -16,7 +16,7 @@ from aiperf.common.environment import Environment from aiperf.metrics.list_metric_aggregation import TDigestListMetricAggregator -from aiperf.metrics.metric_dicts import MetricAggregator, MetricArray +from aiperf.metrics.metric_dicts import MetricAggregator, metric_result_from_array class TestTDigestListMetricAggregator: @@ -133,9 +133,11 @@ def test_to_result_schema_matches_metric_array(self) -> None: digest_agg.extend(values.tolist()) digest_result = digest_agg.to_result(tag="t", header="T", unit="ms") - array_agg = MetricArray() - array_agg.extend(values.tolist()) - array_result = array_agg.to_result(tag="t", header="T", unit="ms") + # Exact numpy-percentile reference (the same computation the t-digest + # approximates). ``metric_result_from_array`` sorts in place, so copy. + array_result = metric_result_from_array( + "t", "T", "ms", values.copy(), float(values.sum()) + ) # Same field set on the Pydantic model. assert set(digest_result.model_fields_set) == set(array_result.model_fields_set) @@ -198,12 +200,9 @@ def test_welford_std_is_stable_on_large_offset_distribution(self) -> None: def test_protocol_runtime_isinstance(self) -> None: """Aggregator should satisfy the ``MetricAggregator`` protocol so - ``MetricsAccumulator`` and ``DerivedSumMetric`` accept both this and - ``MetricArray``.""" + ``MetricsAccumulator`` and ``DerivedSumMetric`` accept it.""" digest_agg = TDigestListMetricAggregator() - array_agg = MetricArray() assert isinstance(digest_agg, MetricAggregator) - assert isinstance(array_agg, MetricAggregator) def test_compression_env_var_flows_to_underlying_sketch( self, monkeypatch: pytest.MonkeyPatch @@ -222,8 +221,7 @@ def test_compression_env_var_flows_to_underlying_sketch( def test_sum_property_for_derived_metric_protocol(self) -> None: """The ``MetricAggregator`` protocol requires a ``sum`` property so - :class:`DerivedSumMetric` can compute uniformly across this and - :class:`MetricArray`.""" + :class:`DerivedSumMetric` can compute the total uniformly.""" agg = TDigestListMetricAggregator() agg.extend([1.0, 2.0, 3.0, 4.0, 5.0]) # Property is exposed and returns the running side-channel sum. diff --git a/tests/unit/metrics/test_metric_array.py b/tests/unit/metrics/test_metric_array.py deleted file mode 100644 index 5287266260..0000000000 --- a/tests/unit/metrics/test_metric_array.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import numpy as np -import pytest - -from aiperf.metrics.metric_dicts import MetricArray - - -@pytest.fixture -def test_array(): - """Create an empty MetricArray for testing.""" - return MetricArray(initial_capacity=5) - - -def assert_array_equal(array: MetricArray, expected_values: list) -> None: - """Assert that an array contains expected values.""" - assert len(array) == len(expected_values) - assert array.sum == sum(expected_values) - np.testing.assert_array_equal(array.data, expected_values) - - -class TestMetricArray: - """Test cases for MetricArray class.""" - - def test_initialization_default_capacity(self): - """Test array initialization with default capacity.""" - array = MetricArray() - assert array.capacity == 10000 - assert len(array) == 0 - assert array.sum == 0 - - def test_initialization_custom_capacity(self): - """Test array initialization with custom capacity.""" - array = MetricArray(initial_capacity=100) - assert array.capacity == 100 - assert len(array) == 0 - - def test_append(self, test_array: MetricArray): - """Test appending single values.""" - test_array.append(1.0) - assert_array_equal(test_array, [1.0]) - - test_array.append(2.5) - assert_array_equal(test_array, [1.0, 2.5]) - - def test_extend(self, test_array: MetricArray): - """Test extending with multiple values.""" - test_array.extend([1.0, 2.0, 3.0]) - assert_array_equal(test_array, [1.0, 2.0, 3.0]) - - test_array.extend([4.0, 5.0]) - assert_array_equal(test_array, [1.0, 2.0, 3.0, 4.0, 5.0]) - - def test_mixed_operations(self, test_array: MetricArray): - """Test mixing append and extend operations.""" - test_array.append(1.0) - test_array.extend([2.0, 3.0]) - test_array.append(4.0) - - assert_array_equal(test_array, [1.0, 2.0, 3.0, 4.0]) - - def test_resize_on_capacity_exceeded(self): - """Test that array resizes when capacity is exceeded.""" - array = MetricArray(initial_capacity=2) - array.extend([1.0, 2.0]) - assert array.capacity == 2 - - # This should trigger resize - array.append(3.0) - assert array.capacity == 4 - assert_array_equal(array, [1.0, 2.0, 3.0]) - - def test_properties(self, test_array: MetricArray): - """Test sum and data properties.""" - values = [1.0, 2.0, 3.0, 4.0] - test_array.extend(values) - assert test_array.sum == 10.0 - assert len(test_array.data) == 4 - np.testing.assert_array_equal(test_array.data, values) - - @pytest.mark.parametrize( - "values,expected_min,expected_max,expected_avg,p50", - [ - ([5.0], 5.0, 5.0, 5.0, 5.0), - ([1.0, 2.0, 3.0, 4.0, 5.0], 1.0, 5.0, 3.0, 3.0), - ([10.0, 20.0, 30.0, 40.0, 50.0], 10.0, 50.0, 30.0, 30.0), - ], - ) - def test_to_result_statistics( - self, - values: list[float], - expected_min: float, - expected_max: float, - expected_avg: float, - p50: float, - ): - """Test to_result method computes statistics correctly.""" - array = MetricArray() - array.extend(values) - - result = array.to_result("test", "Test Metric", "ms") - - assert result.tag == "test" - assert result.header == "Test Metric" - assert result.unit == "ms" - assert result.min == expected_min - assert result.max == expected_max - assert result.avg == expected_avg - assert result.p50 == p50 - assert result.count == len(values) - - -class TestMetricArrayEdgeCases: - """Test edge cases for MetricArray.""" - - def test_array_to_result_raises_error(self): - """Test that to_result raises error for empty array.""" - with pytest.raises(IndexError): - MetricArray().to_result("empty", "Empty Metric", "units") - - @pytest.mark.parametrize("invalid_capacity", [-1, 0]) - def test_invalid_initial_capacity(self, invalid_capacity: int): - """Test behavior with invalid initial capacity.""" - with pytest.raises(ValueError): - MetricArray(initial_capacity=invalid_capacity) - - def test_large_batch_resize(self): - """Test resize behavior when adding large batch.""" - array = MetricArray(initial_capacity=2) - large_batch = list(range(1, 11)) - - array.extend(large_batch) - - # GrowableArray uses doubling strategy, so capacity may be >= 10 - assert array.capacity >= len(large_batch) - assert_array_equal(array, large_batch) diff --git a/tests/unit/metrics/test_output_sequence_length_metric.py b/tests/unit/metrics/test_output_sequence_length_metric.py index 352b4217d8..6224271663 100644 --- a/tests/unit/metrics/test_output_sequence_length_metric.py +++ b/tests/unit/metrics/test_output_sequence_length_metric.py @@ -12,7 +12,6 @@ TotalOutputSequenceLengthMetric, ) from tests.unit.metrics.conftest import ( - create_metric_array, create_record, run_simple_metrics_pipeline, ) @@ -83,7 +82,7 @@ def test_sum_calculation(self, values, expected_sum): """Test that TotalOutputSequenceLengthMetric correctly sums all output tokens""" metric = TotalOutputSequenceLengthMetric() metric_results = MetricResultsDict() - metric_results[OutputSequenceLengthMetric.tag] = create_metric_array(values) + metric_results[OutputSequenceLengthMetric.tag] = sum(values) result = metric.derive_value(metric_results) assert result == expected_sum diff --git a/tests/unit/metrics/test_output_token_count.py b/tests/unit/metrics/test_output_token_count.py index 3736c35c04..e156aeb38f 100644 --- a/tests/unit/metrics/test_output_token_count.py +++ b/tests/unit/metrics/test_output_token_count.py @@ -11,7 +11,6 @@ TotalOutputTokensMetric, ) from tests.unit.metrics.conftest import ( - create_metric_array, create_record, run_simple_metrics_pipeline, ) @@ -76,7 +75,7 @@ def test_sum_calculation(self, values, expected_sum): """Test that TotalOutputTokensMetric correctly sums all output token counts""" metric = TotalOutputTokensMetric() metric_results = MetricResultsDict() - metric_results[OutputTokenCountMetric.tag] = create_metric_array(values) + metric_results[OutputTokenCountMetric.tag] = sum(values) result = metric.derive_value(metric_results) assert result == expected_sum diff --git a/tests/unit/metrics/test_reasoning_token_count.py b/tests/unit/metrics/test_reasoning_token_count.py index 69431ea157..7d1771fe57 100644 --- a/tests/unit/metrics/test_reasoning_token_count.py +++ b/tests/unit/metrics/test_reasoning_token_count.py @@ -11,7 +11,6 @@ TotalReasoningTokensMetric, ) from tests.unit.metrics.conftest import ( - create_metric_array, create_record, run_simple_metrics_pipeline, ) @@ -83,7 +82,7 @@ def test_sum_calculation(self, values, expected_sum): """Test that TotalReasoningTokensMetric correctly sums all reasoning token counts""" metric = TotalReasoningTokensMetric() metric_results = MetricResultsDict() - metric_results[ReasoningTokenCountMetric.tag] = create_metric_array(values) + metric_results[ReasoningTokenCountMetric.tag] = sum(values) result = metric.derive_value(metric_results) assert result == expected_sum diff --git a/tests/unit/metrics/test_replay_sched_lag_metrics.py b/tests/unit/metrics/test_replay_sched_lag_metrics.py index 2ca30da0de..07c33afe86 100644 --- a/tests/unit/metrics/test_replay_sched_lag_metrics.py +++ b/tests/unit/metrics/test_replay_sched_lag_metrics.py @@ -5,35 +5,37 @@ Record-metric tests drive synthetic records that carry an intended schedule timestamp on the dispatched turn (``Turn.timestamp``, ms) and an actual send wall time (``RequestRecord.timestamp_ns``), exactly as fixed-schedule replay -produces. Derived-metric tests build ``MetricResultsDict`` entries in the -production storage shape (``MetricArray`` for record metrics, floats for -already-derived dependencies; see -post_processors/metric_results_processor.py first-touch storage). +produces. + +The ``replay_sched_lag_*`` family is injected post-aggregation from the +``replay_send_schedule_offset`` column (see ``inject_replay_sched_lag_metrics``), +so the derived-metric tests build a :class:`ColumnStore` of offsets and assert on +the injected :class:`MetricResult` values. A golden-parity test pins the injected +values to the exact legacy formula (anchored-percentile over the offsets) that +the deleted ``MetricResultsProcessor`` used, so the columnar port is provably +byte-for-byte equivalent to legacy aiperf for identical offsets. """ -import logging - +import numpy as np import pytest from pytest import param from aiperf.common.constants import NANOS_PER_MILLIS from aiperf.common.enums import MetricFlags from aiperf.common.exceptions import NoMetricValue -from aiperf.common.models import ParsedResponseRecord, Turn +from aiperf.common.models import MetricResult, ParsedResponseRecord, Turn +from aiperf.metrics.column_store import ColumnStore from aiperf.metrics.metric_dicts import MetricResultsDict +from aiperf.metrics.replay_sched_lag_analyzer import inject_replay_sched_lag_metrics from aiperf.metrics.types.replay_sched_lag_metrics import ( + REPLAY_SCHED_DEGRADED_THRESHOLD_MS, ReplaySchedDegradedMetric, ReplaySchedLagP50Metric, ReplaySchedLagP90Metric, ReplaySchedLagP99Metric, ReplaySendScheduleOffsetMetric, - _anchored_lag_ms, -) -from tests.unit.metrics.conftest import ( - create_metric_array, - create_record, - run_simple_metrics_pipeline, ) +from tests.unit.metrics.conftest import create_record, run_simple_metrics_pipeline def _scheduled_record( @@ -46,29 +48,54 @@ def _scheduled_record( return record -def _offset_results(offsets_ms: list[float]) -> MetricResultsDict: - """Results dict holding raw send offsets as production stores them.""" - results = MetricResultsDict() - results[ReplaySendScheduleOffsetMetric.tag] = create_metric_array( - [int(ms * NANOS_PER_MILLIS) for ms in offsets_ms] - ) - return results - - -def _derived_lag_results(offsets_ms: list[float]) -> MetricResultsDict: - """Results dict with the lag percentiles already derived, mirroring - update_derived_metrics' dependency-ordered store-before-dependents.""" - results = _offset_results(offsets_ms) - for metric_cls in ( - ReplaySchedLagP50Metric, - ReplaySchedLagP90Metric, - ReplaySchedLagP99Metric, - ): - results[metric_cls.tag] = metric_cls().derive_value(results) +def _store_with_offsets(offsets_ns: list[int]) -> ColumnStore: + """Build a ColumnStore whose only column is ``replay_send_schedule_offset``, + exactly as the accumulator ingests it during a fixed-schedule replay.""" + store = ColumnStore(initial_capacity=max(len(offsets_ns), 1)) + for idx, offset in enumerate(offsets_ns): + store.ingest( + idx=idx, + record_metrics={ReplaySendScheduleOffsetMetric.tag: offset}, + start_ns=float(idx), + end_ns=float(idx), + generation_start_ns=None, + ) + return store + + +def _inject( + offsets_ms: list[float], + *, + mask: np.ndarray | None = None, + warn=None, +) -> dict[str, MetricResult]: + """Run the injection over a store built from ``offsets_ms`` (ms → ns).""" + store = _store_with_offsets([int(ms * NANOS_PER_MILLIS) for ms in offsets_ms]) + results: dict[str, MetricResult] = {} + inject_replay_sched_lag_metrics(store, results, mask=mask, warn_degraded=warn) return results -def test_family_is_fixed_schedule_only(): +def _legacy_replay_lag(offsets_ns: np.ndarray) -> dict[str, float]: + """The exact computation legacy aiperf's ``MetricResultsProcessor`` used + (``_anchored_lag_ms`` + ``np.percentile`` + degraded threshold). Serves as + the golden reference for the columnar injection port.""" + data = np.asarray(offsets_ns, dtype=np.float64) + anchored = (data - data.min()) / NANOS_PER_MILLIS + p50 = float(np.percentile(anchored, 50.0)) + p90 = float(np.percentile(anchored, 90.0)) + p99 = float(np.percentile(anchored, 99.0)) + return { + ReplaySchedLagP50Metric.tag: p50, + ReplaySchedLagP90Metric.tag: p90, + ReplaySchedLagP99Metric.tag: p99, + ReplaySchedDegradedMetric.tag: float( + int(p99 > REPLAY_SCHED_DEGRADED_THRESHOLD_MS) + ), + } + + +def test_family_is_fixed_schedule_only() -> None: # Turn timestamps reach records in every timing mode, so applicability # must be gated on the run's timing mode, not on timestamp presence. for metric_cls in ( @@ -84,7 +111,7 @@ def test_family_is_fixed_schedule_only(): class TestReplaySendScheduleOffsetMetric: - def test_parse_record_returns_actual_minus_intended_ns(self): + def test_parse_record_returns_actual_minus_intended_ns(self) -> None: records = [_scheduled_record(intended_ms=2, actual_send_ms=5)] results = run_simple_metrics_pipeline( @@ -93,7 +120,7 @@ def test_parse_record_returns_actual_minus_intended_ns(self): assert results[ReplaySendScheduleOffsetMetric.tag] == [3 * NANOS_PER_MILLIS] - def test_parse_record_no_turn_timestamp_yields_no_value(self): + def test_parse_record_no_turn_timestamp_yields_no_value(self) -> None: records = [_scheduled_record(intended_ms=None, actual_send_ms=5)] results = run_simple_metrics_pipeline( @@ -103,84 +130,109 @@ def test_parse_record_no_turn_timestamp_yields_no_value(self): assert ReplaySendScheduleOffsetMetric.tag not in results -class TestReplaySchedLagPercentiles: - def test_derive_value_anchors_at_least_late_request(self): +class TestInjectReplaySchedLagPercentiles: + def test_injection_anchors_at_least_late_request(self) -> None: # Send offsets of 5/5/15/105 ms anchor to [0, 0, 10, 100] ms (the # least-late requests define zero). - results = _offset_results([5, 5, 15, 105]) - - # np.percentile (linear interpolation) over [0, 0, 10, 100]. - assert ReplaySchedLagP50Metric().derive_value(results) == pytest.approx(5.0) - assert ReplaySchedLagP90Metric().derive_value(results) == pytest.approx(73.0) - assert ReplaySchedLagP99Metric().derive_value(results) == pytest.approx(97.3) - - def test_anchored_lag_cached_per_array_and_invalidated_on_append(self): - # p50/p90/p99 derive back-to-back from the same offsets: the second - # call returns the cached array; appending (a realtime tick) or a - # different offsets object recomputes. - results = _offset_results([5, 15]) - first = _anchored_lag_ms(results) - assert _anchored_lag_ms(results) is first - - results[ReplaySendScheduleOffsetMetric.tag].append(25 * NANOS_PER_MILLIS) - grown = _anchored_lag_ms(results) - assert grown is not first - assert list(grown) == [0.0, 10.0, 20.0] - - other = _offset_results([5, 15]) - assert _anchored_lag_ms(other) is not grown - - def test_derive_value_uniform_lag_reports_zero(self): + results = _inject([5, 5, 15, 105]) + + assert results[ReplaySchedLagP50Metric.tag].avg == pytest.approx(5.0) + assert results[ReplaySchedLagP90Metric.tag].avg == pytest.approx(73.0) + assert results[ReplaySchedLagP99Metric.tag].avg == pytest.approx(97.3) + + def test_injection_uniform_lag_reports_zero(self) -> None: # A constant delay on every request is invisible after anchoring -- # the documented limitation of the post-hoc approximation. - results = _offset_results([700, 700, 700]) + results = _inject([700, 700, 700]) - assert ReplaySchedLagP99Metric().derive_value(results) == 0.0 + assert results[ReplaySchedLagP99Metric.tag].avg == 0.0 - def test_derive_value_no_offsets_raises_no_metric_value(self): - with pytest.raises(NoMetricValue): - ReplaySchedLagP99Metric().derive_value(MetricResultsDict()) + def test_injection_noop_when_column_absent(self) -> None: + results: dict[str, MetricResult] = {} + inject_replay_sched_lag_metrics(ColumnStore(initial_capacity=4), results) + assert results == {} - def test_derive_value_empty_offset_array_raises_no_metric_value(self): - with pytest.raises(NoMetricValue): - ReplaySchedLagP99Metric().derive_value(_offset_results([])) + def test_injection_noop_when_all_offsets_masked_out(self) -> None: + results = _inject([5, 15], mask=np.array([False, False])) + assert results == {} + + def test_mask_selects_subset(self) -> None: + # Only the first two offsets (0, 10 ms anchored) count; the 500 ms tail + # is masked out so it cannot inflate the percentiles. + results = _inject([0, 10, 510], mask=np.array([True, True, False])) + assert results[ReplaySchedLagP99Metric.tag].avg == pytest.approx(9.9) -class TestReplaySchedDegradedMetric: +class TestInjectReplaySchedDegraded: @pytest.mark.parametrize( "tail_lag_ms, expected", [ - param(400, 0, id="below_threshold"), - param(500, 0, id="at_threshold_not_degraded"), - param(600, 1, id="above_threshold"), + param(400, 0.0, id="below_threshold"), + param(500, 0.0, id="at_threshold_not_degraded"), + param(600, 1.0, id="above_threshold"), ], ) # fmt: skip - def test_derive_value_flags_p99_over_threshold( - self, tail_lag_ms: int, expected: int - ): + def test_degraded_flags_p99_over_threshold( + self, tail_lag_ms: int, expected: float + ) -> None: # Half the requests on time, half late by tail_lag_ms: p99 == tail_lag_ms. - results = _derived_lag_results([0.0] * 5 + [float(tail_lag_ms)] * 5) + results = _inject([0.0] * 5 + [float(tail_lag_ms)] * 5) + + assert results[ReplaySchedDegradedMetric.tag].avg == expected + + def test_warn_degraded_called_once_with_percentiles(self) -> None: + calls: list[tuple[float, float, float]] = [] + results = _inject([0.0] * 5 + [600.0] * 5, warn=lambda *a: calls.append(a)) + + assert results[ReplaySchedDegradedMetric.tag].avg == 1.0 + assert len(calls) == 1 + p50, p90, p99 = calls[0] + assert p99 == pytest.approx(600.0) - assert ReplaySchedDegradedMetric().derive_value(results) == expected + def test_warn_degraded_not_called_when_healthy(self) -> None: + calls: list[tuple[float, float, float]] = [] + _inject([0.0, 10.0, 20.0], warn=lambda *a: calls.append(a)) + assert calls == [] - def test_derive_value_missing_percentiles_raises_no_metric_value(self): + +class TestDeferredDerivation: + """The scalar summarize path cannot see the full offset array, so every + metric in the family defers; the injection fills the values instead.""" + + @pytest.mark.parametrize( + "metric_cls", + [ + ReplaySchedLagP50Metric, + ReplaySchedLagP90Metric, + ReplaySchedLagP99Metric, + ReplaySchedDegradedMetric, + ], + ) # fmt: skip + def test_derive_value_always_defers(self, metric_cls) -> None: with pytest.raises(NoMetricValue): - ReplaySchedDegradedMetric().derive_value(MetricResultsDict()) + metric_cls()._derive_value(MetricResultsDict()) - def test_degraded_warning_logged_once_per_run( - self, caplog: pytest.LogCaptureFixture - ): - # summarize() re-derives every realtime tick; the warning must not - # repeat while the metric value stays continuous. - results = _derived_lag_results([0.0] * 5 + [600.0] * 5) - metric = ReplaySchedDegradedMetric() - - with caplog.at_level(logging.WARNING): - assert metric.derive_value(results) == 1 - assert metric.derive_value(results) == 1 - assert metric.derive_value(results) == 1 - - warnings = [ - r for r in caplog.records if "Replay schedule degraded" in r.message - ] - assert len(warnings) == 1 + +class TestGoldenParityVsLegacy: + """Prove the columnar injection is byte-for-byte identical to the legacy + ``MetricResultsProcessor`` derive formula for the same per-record offsets.""" + + @pytest.mark.parametrize("seed", [0, 1, 7, 42, 1234]) + def test_injection_matches_legacy_formula(self, seed: int) -> None: + rng = np.random.default_rng(seed) + # Mixed magnitudes incl. a heavy tail so p99 straddles the degraded + # threshold across seeds; offsets are epoch-scale ns like production. + base = 1_700_000_000_000_000_000 + jitter_ns = rng.integers(0, 900_000_000, size=200) # up to 900 ms late + offsets_ns = (base + jitter_ns).astype(np.int64) + + golden = _legacy_replay_lag(offsets_ns) + + store = _store_with_offsets(offsets_ns.tolist()) + results: dict[str, MetricResult] = {} + inject_replay_sched_lag_metrics(store, results) + + for tag, expected in golden.items(): + assert results[tag].avg == pytest.approx(expected, abs=1e-9), ( + f"{tag}: injection {results[tag].avg} != legacy {expected}" + ) From b8b436fa2b63599cc7e4fd293db0a692a8d2103b Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 18:05:00 -0700 Subject: [PATCH 06/39] refactor(metrics): delete legacy pre-columnar aggregate machinery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following a reachability audit of the metrics subsystem against the columnar accumulator/analyzer architecture, remove dead code left over from the old per-instance results-processor era. No behavior change (full unit suite + live mock run green; benchmark_duration/request_count/min/max verified). - BaseAggregateMetric: drop the per-instance fold scaffolding (`__init__` default_value/_value seeding, `aggregate_value` binding, `current_value` property, abstract `_aggregate_value`). MetricsAccumulator folds AGGREGATE columns purely via `aggregation_kind` (SUM/MIN/MAX) — the running-state fold was never called in the columnar path. Remove the now-dead `_aggregate_value` overrides in base_aggregate_counter_metric, min_request_metric (+ its sys.maxsize __init__ seed), max_response_metric, and the accuracy transport metrics; MIN/MAX already declare aggregation_kind. - DerivedSumMetric: drop the dead `isinstance(value, MetricAggregator)` branch — the accumulator only ever stores scalar sums in scalar_dict now. - MetricsAccumulator.full_metrics(): delete unused test-only accessor (not part of AccumulatorProtocol). - Port the run_simple_metrics_pipeline test helper to fold via aggregation_kind (mirroring the accumulator) instead of the removed aggregate_value/ current_value; rewrite the min/max metric tests accordingly. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/metrics/accumulator.py | 4 -- .../metrics/base_aggregate_counter_metric.py | 4 -- src/aiperf/metrics/base_aggregate_metric.py | 68 ++++--------------- src/aiperf/metrics/derived_sum_metric.py | 7 +- src/aiperf/metrics/types/accuracy_metrics.py | 6 -- .../metrics/types/max_response_metric.py | 5 -- .../metrics/types/min_request_metric.py | 16 ++--- tests/unit/metrics/conftest.py | 29 ++++++-- .../unit/metrics/test_max_response_metric.py | 37 +++------- tests/unit/metrics/test_min_request_metric.py | 30 +++----- .../test_metric_results_processor.py | 14 ---- .../test_metrics_accumulator.py | 22 ------ 12 files changed, 67 insertions(+), 175 deletions(-) diff --git a/src/aiperf/metrics/accumulator.py b/src/aiperf/metrics/accumulator.py index 92fb7befc0..1b4fea1493 100644 --- a/src/aiperf/metrics/accumulator.py +++ b/src/aiperf/metrics/accumulator.py @@ -735,7 +735,3 @@ def _compute_timeslices( ) return timeslices - - async def full_metrics(self) -> dict[MetricTagT, MetricResult]: - """Returns the full metrics results, including derived metrics.""" - return self._compute_results() diff --git a/src/aiperf/metrics/base_aggregate_counter_metric.py b/src/aiperf/metrics/base_aggregate_counter_metric.py index fbe94c6937..65ea282d4b 100644 --- a/src/aiperf/metrics/base_aggregate_counter_metric.py +++ b/src/aiperf/metrics/base_aggregate_counter_metric.py @@ -33,7 +33,3 @@ def _parse_record( ) -> MetricValueTypeVarT: """Return the value of the counter for this record.""" return 1 # type: ignore - - def _aggregate_value(self, value: MetricValueTypeVarT) -> None: - """Aggregate the metric value.""" - self._value += value # type: ignore diff --git a/src/aiperf/metrics/base_aggregate_metric.py b/src/aiperf/metrics/base_aggregate_metric.py index 4f9b8f5214..1138c98258 100644 --- a/src/aiperf/metrics/base_aggregate_metric.py +++ b/src/aiperf/metrics/base_aggregate_metric.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from abc import ABC, abstractmethod -from collections.abc import Callable from typing import ClassVar, Generic from aiperf.common.enums import AggregationKind, MetricType, MetricValueTypeVarT @@ -13,54 +12,28 @@ class BaseAggregateMetric( Generic[MetricValueTypeVarT], BaseMetric[MetricValueTypeVarT], ABC ): - """A base class for aggregate metrics. These metrics keep track of a value or list of values over time. + """A base class for aggregate metrics. - This metric type is unique in the fact that it is split into 2 distinct phases of processing, in order to support distributed processing. - - For each distributed RecordProcessor, an instance of this class is created. This instance is passed the record and the existing record metrics, - and is responsible for returning the individual value for that record. It should not use or update the aggregate value here. - - The MetricsAccumulator folds the emitted per-record aggregate values into a - single scalar. Subclasses declare ``aggregation_kind`` (SUM/MAX/MIN) so the - vectorized path can combine values without replaying ``_aggregate_value``. - The default is SUM. + Each distributed RecordProcessor emits the per-record value via + ``_parse_record`` (this record alone; it must not fold across records). The + MetricsAccumulator stores those values in a numpy column and folds them into + a single scalar according to the subclass's ``aggregation_kind`` + (SUM/MAX/MIN, default SUM) — no per-instance running state is kept. Examples: ```python class RequestCountMetric(BaseAggregateMetric[int]): - # ... Metric attributes ... + aggregation_kind = AggregationKind.SUM def _parse_record(self, record: ParsedResponseRecord, record_metrics: MetricRecordDict) -> int: - # We just return 1 since we are tracking the total count, and this is a single request. + # One per request; the accumulator sums the column. return 1 - - def _aggregate_value(self, value: int) -> None: - # We add the value to the aggregate value. - self._value += value ``` """ type = MetricType.AGGREGATE aggregation_kind: ClassVar[AggregationKind] = AggregationKind.SUM - def __init__(self, default_value: MetricValueTypeVarT | None = None) -> None: - """Initialize the metric with optionally with a default value. If no default value is provided, - the default value is automatically set based on the value type.""" - self._value: MetricValueTypeVarT = ( # type: ignore - default_value - if default_value is not None - else self.value_type.default_factory() - ) - self.aggregate_value: Callable[[MetricValueTypeVarT], None] = ( - self._aggregate_value - ) - super().__init__() - - @property - def current_value(self) -> MetricValueTypeVarT: - """Get the current value of the metric.""" - return self._value - def parse_record( self, record: ParsedResponseRecord, record_metrics: MetricRecordDict ) -> MetricValueTypeVarT: @@ -77,29 +50,16 @@ def parse_record( def _parse_record( self, record: ParsedResponseRecord, record_metrics: MetricRecordDict ) -> MetricValueTypeVarT: - """Parse the record and *return* the individual value base on this record, and this record alone. This - method is implemented by subclasses. + """Parse the record and *return* the individual value based on this record, + and this record alone. Implemented by subclasses. - NOTE: Do not use or update the aggregate value here. + NOTE: Do not fold across records here — the accumulator combines the + per-record values via ``aggregation_kind``. - This method is called after the required metrics are checked, so it can assume that the required metrics are available. - This method is called after the record is checked, so it can assume that the record is valid. + Called after the required metrics and record are validated, so it can + assume both are available/valid. Raises: ValueError: If the metric cannot be computed for the given inputs. """ raise NotImplementedError("Subclasses must implement this method") - - # NOTE: This method does not return a value on purpose, as a hint to the user that the - # internal value is supposed to be updated. - @abstractmethod - def _aggregate_value(self, value: MetricValueTypeVarT) -> None: - """Aggregate the metric value. This method is implemented by subclasses. - - This method is called with the result value from the `_parse_record` method, from each distributed record processor. - It is the responsibility of each metric class to implement how values from different processes are aggregated, such - as summing the values, or taking the min/max/average, etc. - - NOTE: The order of the values is not guaranteed. - """ - raise NotImplementedError("Subclasses must implement this method") diff --git a/src/aiperf/metrics/derived_sum_metric.py b/src/aiperf/metrics/derived_sum_metric.py index 52c710772c..de9f609c4d 100644 --- a/src/aiperf/metrics/derived_sum_metric.py +++ b/src/aiperf/metrics/derived_sum_metric.py @@ -6,7 +6,7 @@ from aiperf.common.enums import MetricFlags, MetricValueTypeVarT from aiperf.metrics.base_derived_metric import BaseDerivedMetric from aiperf.metrics.base_record_metric import BaseRecordMetric -from aiperf.metrics.metric_dicts import MetricAggregator, MetricResultsDict +from aiperf.metrics.metric_dicts import MetricResultsDict RecordMetricT = TypeVar("RecordMetricT", bound=BaseRecordMetric) @@ -58,8 +58,5 @@ def _derive_value(cls, metric_results: MetricResultsDict) -> MetricValueTypeVarT raise ValueError(f"{cls.record_metric_type.tag} is missing in the metrics.") # MetricsAccumulator stores the running-sum scalar in ``scalar_dict[tag]`` # (see ``MetricsAccumulator._collect_scalars_and_arrays``), so the value - # already IS the sum. The ``MetricAggregator`` branch supports direct unit - # tests and older callers that still pass a live aggregate container. - if isinstance(value, MetricAggregator): - return value.sum + # already IS the sum. return value diff --git a/src/aiperf/metrics/types/accuracy_metrics.py b/src/aiperf/metrics/types/accuracy_metrics.py index 84be54db7e..aa2c52a044 100644 --- a/src/aiperf/metrics/types/accuracy_metrics.py +++ b/src/aiperf/metrics/types/accuracy_metrics.py @@ -43,9 +43,6 @@ def _parse_record( raise NoMetricValue(f"{ACCURACY_RECORD_CORRECT_KEY} not in record_metrics") return float(value) - def _aggregate_value(self, value: float) -> None: - self._value += value - class AccuracyUnparsedSumMetric(BaseAggregateMetric[float]): """Registration for the per-record ``accuracy_unparsed`` transport key. @@ -73,6 +70,3 @@ def _parse_record( if value is None: raise NoMetricValue(f"{ACCURACY_RECORD_UNPARSED_KEY} not in record_metrics") return float(value) - - def _aggregate_value(self, value: float) -> None: - self._value += value diff --git a/src/aiperf/metrics/types/max_response_metric.py b/src/aiperf/metrics/types/max_response_metric.py index 6a6c2a2eb0..0aaa9ffcb1 100644 --- a/src/aiperf/metrics/types/max_response_metric.py +++ b/src/aiperf/metrics/types/max_response_metric.py @@ -47,8 +47,3 @@ def _parse_record( request_latency: int = record_metrics.get_or_raise(RequestLatencyMetric) # type: ignore final_response_ts = record.timestamp_ns + request_latency return final_response_ts - - def _aggregate_value(self, value: int) -> None: - """Aggregate the metric value. For this metric, we just take the max of the values from the different processes.""" - if value > self._value: - self._value = value diff --git a/src/aiperf/metrics/types/min_request_metric.py b/src/aiperf/metrics/types/min_request_metric.py index a08adb81c0..9ceb914f6f 100644 --- a/src/aiperf/metrics/types/min_request_metric.py +++ b/src/aiperf/metrics/types/min_request_metric.py @@ -1,7 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import sys - from aiperf.common.enums import ( AggregationKind, MetricConsoleGroup, @@ -31,20 +29,14 @@ class MinRequestTimestampMetric(BaseAggregateMetric[int]): aggregation_kind = AggregationKind.MIN required_metrics = None - def __init__(self) -> None: - # Default to a large value, so that any request timestamp will be smaller. - super().__init__(default_value=sys.maxsize) - def _parse_record( self, record: ParsedResponseRecord, record_metrics: MetricRecordDict, ) -> int: - """Return the request timestamp.""" + """Return the request timestamp. + + The accumulator folds these to the run minimum via ``aggregation_kind``. + """ # NOTE: Use the request timestamp_ns, not the start_perf_ns, because we want wall-clock timestamps, return record.timestamp_ns - - def _aggregate_value(self, value: int) -> None: - """Aggregate the metric value. For this metric, we just take the min of the values from the different processes.""" - if value < self._value: - self._value = value diff --git a/tests/unit/metrics/conftest.py b/tests/unit/metrics/conftest.py index 3bd0015611..5068a3f850 100644 --- a/tests/unit/metrics/conftest.py +++ b/tests/unit/metrics/conftest.py @@ -5,7 +5,12 @@ """ -from aiperf.common.enums import CreditPhase, MetricType, ModelSelectionStrategy +from aiperf.common.enums import ( + AggregationKind, + CreditPhase, + MetricType, + ModelSelectionStrategy, +) from aiperf.common.exceptions import NoMetricValue from aiperf.common.models import ( ErrorDetails, @@ -119,6 +124,9 @@ def run_simple_metrics_pipeline( ] metric_results = MetricResultsDict() + # Per-record AGGREGATE values, folded once at the end via aggregation_kind + # exactly as MetricsAccumulator combines the numpy column. + aggregate_values: dict[MetricTagT, list] = {} for record in records: # STAGE 1: Parse the metrics for each record, and store the values in a dict metric_dict = MetricRecordDict() @@ -131,17 +139,30 @@ def run_simple_metrics_pipeline( # If a metric can't be calculated, skip it for this record pass - # STAGE 2: Aggregate the values of the aggregate metrics, and append new values for record metrics + # STAGE 2: Collect per-record aggregate values; append record-metric values. for metric in metrics: if metric.type == MetricType.AGGREGATE: if metric.tag in metric_dict: - metric.aggregate_value(metric_dict[metric.tag]) - metric_results[metric.tag] = metric.current_value + aggregate_values.setdefault(metric.tag, []).append( + metric_dict[metric.tag] + ) elif metric.type == MetricType.RECORD and metric.tag in metric_dict: metric_results.setdefault(metric.tag, []).append( metric_dict[metric.tag] ) + # STAGE 2b: Fold the aggregate columns via aggregation_kind (SUM/MIN/MAX). + _fold = { + AggregationKind.SUM: sum, + AggregationKind.MIN: min, + AggregationKind.MAX: max, + } + for metric in metrics: + if metric.type == MetricType.AGGREGATE and metric.tag in aggregate_values: + metric_results[metric.tag] = _fold[metric.aggregation_kind]( + aggregate_values[metric.tag] + ) + # STAGE 3: Compute all of the derived metrics for metric in metrics: if metric.type == MetricType.DERIVED: diff --git a/tests/unit/metrics/test_max_response_metric.py b/tests/unit/metrics/test_max_response_metric.py index 99483697c0..9317823ba6 100644 --- a/tests/unit/metrics/test_max_response_metric.py +++ b/tests/unit/metrics/test_max_response_metric.py @@ -3,10 +3,11 @@ from pytest import approx +from aiperf.common.enums import AggregationKind from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.metrics.types.max_response_metric import MaxResponseTimestampMetric from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric -from tests.unit.metrics.conftest import create_record +from tests.unit.metrics.conftest import create_record, run_simple_metrics_pipeline class TestMaxResponseTimestampMetric: @@ -30,36 +31,20 @@ def test_max_response_calculation(self): expected = 100 + 50 # start_ns + latency = timestamp_ns + latency assert result == approx(expected) - def test_max_response_aggregation(self): - """Test aggregation finds maximum response timestamp""" + def test_uses_max_aggregation_kind(self) -> None: + """The accumulator folds the per-record response timestamps to the run max.""" + assert MaxResponseTimestampMetric.aggregation_kind == AggregationKind.MAX + + def test_max_response_aggregation(self) -> None: + """Per-record final-response timestamps fold to the maximum via aggregation_kind.""" records = [ create_record(start_ns=100, responses=[150]), # latency: 50, final: 150 - create_record( - start_ns=200, responses=[300] - ), # latency: 100, final: 300 (max) + create_record(start_ns=200, responses=[300]), # latency: 100, final: 300 create_record( start_ns=300, responses=[350] ), # latency: 50, final: 350 (max) ] - request_latency_metric = RequestLatencyMetric() - max_response_metric = MaxResponseTimestampMetric() - - # Process each record and aggregate - for record in records: - # Get request latency - latency = request_latency_metric.parse_record(record, MetricRecordDict()) - - # Get max response timestamp - metrics_dict = MetricRecordDict() - metrics_dict[RequestLatencyMetric.tag] = latency - value = max_response_metric.parse_record(record, metrics_dict) - max_response_metric.aggregate_value(value) - - # Should have the maximum final response timestamp - assert max_response_metric.current_value == approx(350) + results = run_simple_metrics_pipeline(records, MaxResponseTimestampMetric.tag) - def test_max_response_default_value(self): - """Test that default value is zero""" - metric = MaxResponseTimestampMetric() - assert metric.current_value == 0 + assert results[MaxResponseTimestampMetric.tag] == approx(350) diff --git a/tests/unit/metrics/test_min_request_metric.py b/tests/unit/metrics/test_min_request_metric.py index 736ace99cc..d8b883f964 100644 --- a/tests/unit/metrics/test_min_request_metric.py +++ b/tests/unit/metrics/test_min_request_metric.py @@ -1,15 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import sys - +from aiperf.common.enums import AggregationKind from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.metrics.types.min_request_metric import MinRequestTimestampMetric -from tests.unit.metrics.conftest import create_record +from tests.unit.metrics.conftest import create_record, run_simple_metrics_pipeline class TestMinRequestTimestampMetric: - def test_min_request_timestamp(self): + def test_min_request_timestamp(self) -> None: """Test min request timestamp extraction""" record = create_record(start_ns=1500) @@ -17,25 +16,18 @@ def test_min_request_timestamp(self): result = metric.parse_record(record, MetricRecordDict()) assert result == 1500 # Uses timestamp_ns which equals start_ns - def test_min_request_aggregation(self): - """Test aggregation finds minimum request timestamp""" + def test_uses_min_aggregation_kind(self) -> None: + """The accumulator folds the per-record timestamps to the run minimum.""" + assert MinRequestTimestampMetric.aggregation_kind == AggregationKind.MIN + + def test_min_request_aggregation(self) -> None: + """Per-record parse values fold to the minimum via aggregation_kind.""" records = [ create_record(start_ns=2000), # timestamp: 2000 create_record(start_ns=1000), # timestamp: 1000 (minimum) create_record(start_ns=3000), # timestamp: 3000 ] - metric = MinRequestTimestampMetric() - - # Process each record and aggregate - for record in records: - timestamp = metric.parse_record(record, MetricRecordDict()) - metric.aggregate_value(timestamp) - - # Should have the minimum timestamp - assert metric.current_value == 1000 + results = run_simple_metrics_pipeline(records, MinRequestTimestampMetric.tag) - def test_min_request_default_value(self): - """Test that default value is max int size""" - metric = MinRequestTimestampMetric() - assert metric.current_value == sys.maxsize + assert results[MinRequestTimestampMetric.tag] == 1000 diff --git a/tests/unit/post_processors/test_metric_results_processor.py b/tests/unit/post_processors/test_metric_results_processor.py index 3812de5dd7..def3a0ae34 100644 --- a/tests/unit/post_processors/test_metric_results_processor.py +++ b/tests/unit/post_processors/test_metric_results_processor.py @@ -169,17 +169,3 @@ async def test_summarize_returns_typed_summary(self, mock_run) -> None: assert summary.results[RequestLatencyMetric.tag].unit == "ms" assert summary.results[RequestLatencyMetric.tag].avg == pytest.approx(42.0) - @pytest.mark.asyncio - async def test_full_metrics_returns_raw_metric_results(self, mock_run) -> None: - accumulator = MetricsAccumulator(mock_run) - await accumulator.process_record( - create_metric_records_message( - x_request_id="test-1", - results=[{RequestLatencyMetric.tag: 42_000_000.0}], - ).to_data() - ) - - full_results = await accumulator.full_metrics() - - assert RequestLatencyMetric.tag in full_results - assert full_results[RequestLatencyMetric.tag].avg == pytest.approx(42_000_000.0) diff --git a/tests/unit/post_processors/test_metrics_accumulator.py b/tests/unit/post_processors/test_metrics_accumulator.py index d92644496d..4be047d8e9 100644 --- a/tests/unit/post_processors/test_metrics_accumulator.py +++ b/tests/unit/post_processors/test_metrics_accumulator.py @@ -924,28 +924,6 @@ def test_summary_satisfies_accumulator_result(self) -> None: assert isinstance(summary, AccumulatorResult) -class TestFullMetrics: - @pytest.mark.asyncio - async def test_full_metrics_with_derived( - self, mock_metric_registry: Mock, mock_run - ) -> None: - """Test full_metrics returns the complete results dict including derived metrics.""" - - def mock_derive_func(results_dict: MetricResultsDict) -> float: - return 200.0 - - processor = MetricsAccumulator(mock_run) - processor._derive_funcs = {RequestThroughputMetric.tag: mock_derive_func} - processor._metric_classes = { - RequestThroughputMetric.tag: RequestThroughputMetric - } - - full_results = await processor.full_metrics() - assert RequestThroughputMetric.tag in full_results - assert isinstance(full_results[RequestThroughputMetric.tag], MetricResult) - assert full_results[RequestThroughputMetric.tag].avg == 200.0 - - class TestMetricResultFromArray: """Test metric_result_from_array computes correct statistics.""" From ee9fa97c93b812482aac97604b6d8c8295d42f93 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 18:15:14 -0700 Subject: [PATCH 07/39] docs: regenerate cli-options.md after merge (cyclopts list-default rendering) Post-merge regeneration: #1148 stopped emitting `_Default: []_` for list-type flags; the branch's committed copy still had them. `make generate-cli-docs` output, no manual edits. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/cli-options.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/cli-options.md b/docs/cli-options.md index c1dcac83f5..f3de59ee92 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -265,7 +265,6 @@ aiperf profile --model your_model --url localhost:8000 --goodput "request_latenc #### `-m`, `--model-names`, `--model` `` Model name(s) to be benchmarked. Can be a comma-separated list or a single model name. -
_Default: `[]`_ #### `--model-selection-strategy` `` @@ -427,7 +426,6 @@ HuggingFace dataset subset/config name to override the plugin default (e.g. `sha #### `--dataset-filter` `` Dataset-specific filter in key=value form. Repeat for multiple filters. Only supported by public datasets that declare filter support. -
_Default: `[]`_ #### `--custom-dataset-type` `` @@ -1375,7 +1373,6 @@ Deprecated and ignored. The bayesian preset and the optuna expert mode both use #### `--variant`, `--sweep-variant` `` Repeatable: each occurrence describes one sweep variation. Format: '[name:] key=value, key=value, ...'. Keys are CLI flag names with the leading '--' stripped, in either kebab-case or snake_case (isl, osl, concurrency, request-rate / request_rate, request-count / request_count, benchmark-duration / benchmark_duration, ...). Multi-occurrence emits a ScenarioSweep. Mutually exclusive with magic-list flags, --search-recipe, and YAML-declared sweeps. Single-occurrence is rejected -- use the standalone --isl / --osl / --concurrency flags for a one-off. -
_Default: `[]`_ #### `--search-sla` `` @@ -1726,7 +1723,6 @@ HTTP port for health endpoints (/healthz, /readyz). Required for Kubernetes live #### `-m`, `--model-names`, `--model` `` Model name(s) to be benchmarked. Can be a comma-separated list or a single model name. -
_Default: `[]`_ #### `--model-selection-strategy` `` @@ -1888,7 +1884,6 @@ HuggingFace dataset subset/config name to override the plugin default (e.g. `sha #### `--dataset-filter` `` Dataset-specific filter in key=value form. Repeat for multiple filters. Only supported by public datasets that declare filter support. -
_Default: `[]`_ #### `--custom-dataset-type` `` @@ -2836,7 +2831,6 @@ Deprecated and ignored. The bayesian preset and the optuna expert mode both use #### `--variant`, `--sweep-variant` `` Repeatable: each occurrence describes one sweep variation. Format: '[name:] key=value, key=value, ...'. Keys are CLI flag names with the leading '--' stripped, in either kebab-case or snake_case (isl, osl, concurrency, request-rate / request_rate, request-count / request_count, benchmark-duration / benchmark_duration, ...). Multi-occurrence emits a ScenarioSweep. Mutually exclusive with magic-list flags, --search-recipe, and YAML-declared sweeps. Single-occurrence is rejected -- use the standalone --isl / --osl / --concurrency flags for a one-off. -
_Default: `[]`_ #### `--search-sla` `` From 61b7d82da1695311a07a1e86ffb41f28f150ee6b Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 18:15:14 -0700 Subject: [PATCH 08/39] refactor(tests/docs): consolidate accumulator tests; scrub stale legacy references - Merge the accumulator tests from the legacy-named test_metric_results_processor.py (which tested MetricsAccumulator, not the deleted MetricResultsProcessor) into test_metrics_accumulator.py and delete the misnamed file. - Fix docs that still described the retired storage/aggregation model: list-metric-aggregation.md (MetricArray -> ColumnStore numpy column) and the metrics-flow diagram (per-instance aggregate_value fold -> aggregation_kind column fold). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/diagrams/metrics-flow.md | 2 +- docs/reference/list-metric-aggregation.md | 2 +- .../test_metric_results_processor.py | 171 ------------------ .../test_metrics_accumulator.py | 138 ++++++++++++++ 4 files changed, 140 insertions(+), 173 deletions(-) delete mode 100644 tests/unit/post_processors/test_metric_results_processor.py diff --git a/docs/diagrams/metrics-flow.md b/docs/diagrams/metrics-flow.md index 30c7cf82b4..a5f8f34647 100644 --- a/docs/diagrams/metrics-flow.md +++ b/docs/diagrams/metrics-flow.md @@ -41,7 +41,7 @@ flowchart TD G --> H1["RECORD Collection
append(125ms)
append(87ms)
append(203ms)
(Collect all individual values)"] %% AGGREGATE Processing in Central - G --> H2["AGGREGATE Accumulation
aggregate_value(+1) → total=1
aggregate_value(+1) → total=2
aggregate_value(+1) → total=3
(Accumulate across processors)"] + G --> H2["AGGREGATE Column
append(1) append(1) append(1)
fold via aggregation_kind (SUM) → 3
(Numpy column folded in MetricsAccumulator)"] H1 --> L["MetricResultsDict
(Full profile run results)"] H2 --> L diff --git a/docs/reference/list-metric-aggregation.md b/docs/reference/list-metric-aggregation.md index 2f75fb9851..2a405b55e9 100644 --- a/docs/reference/list-metric-aggregation.md +++ b/docs/reference/list-metric-aggregation.md @@ -51,7 +51,7 @@ For ICL specifically: - `count`, `sum`, `min`, `max`, `avg`, `std` are computed exactly and match what an exact array would produce. - Per-request ICL lists in `profile_export.jsonl` are unchanged — anything that needs sample-level precision can read those. -For all other metrics: **no change**. Scalar record metrics still use the exact-storage `MetricArray` path. Aggregate metrics (`inter_token_latency`, `request_latency`, etc.) compute through their own existing aggregator; t-digest is not in their path. +For all other metrics: **no change**. Scalar record metrics still use the exact numpy column store (`ColumnStore`). Aggregate metrics (`inter_token_latency`, `request_latency`, etc.) compute through their own existing aggregator; t-digest is not in their path. ## Where it lives diff --git a/tests/unit/post_processors/test_metric_results_processor.py b/tests/unit/post_processors/test_metric_results_processor.py deleted file mode 100644 index def3a0ae34..0000000000 --- a/tests/unit/post_processors/test_metric_results_processor.py +++ /dev/null @@ -1,171 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -from unittest.mock import patch - -import pytest - -from aiperf.common.constants import NANOS_PER_SECOND -from aiperf.common.exceptions import NoMetricValue -from aiperf.metrics.accumulator import MetricsAccumulator -from aiperf.metrics.accumulator_models import AccumulatorMetricsSummary -from aiperf.metrics.metric_dicts import MetricResultsDict -from aiperf.metrics.types.inter_chunk_latency_metric import InterChunkLatencyMetric -from aiperf.metrics.types.max_response_metric import MaxResponseTimestampMetric -from aiperf.metrics.types.min_request_metric import MinRequestTimestampMetric -from aiperf.metrics.types.request_count_metric import RequestCountMetric -from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric -from aiperf.metrics.types.request_throughput_metric import RequestThroughputMetric -from tests.unit.post_processors.conftest import create_metric_records_message - - -class TestMetricsAccumulator: - """Tests for the accumulator-backed metric summary engine.""" - - def test_initialization(self, mock_run) -> None: - accumulator = MetricsAccumulator(mock_run) - - assert accumulator.record_count == 0 - assert accumulator.column_store.count == 0 - assert accumulator._network_rtt_ns is None - assert isinstance(accumulator._derive_funcs, dict) - assert isinstance(accumulator._tags_to_types, dict) - assert isinstance(accumulator._metric_classes, dict) - - @pytest.mark.asyncio - async def test_process_record_metric_accumulates_values(self, mock_run) -> None: - accumulator = MetricsAccumulator(mock_run) - - for idx, value in enumerate([42_000_000.0, 84_000_000.0]): - message = create_metric_records_message( - x_request_id=f"test-{idx}", - request_start_ns=1_000_000_000 + idx, - request_end_ns=1_100_000_000 + idx, - results=[{RequestLatencyMetric.tag: value}], - ) - await accumulator.process_record(message.to_data()) - - summary = await accumulator.summarize() - result = summary.results[RequestLatencyMetric.tag] - - assert accumulator.record_count == 2 - assert result.unit == "ms" - assert result.count == 2 - assert result.avg == pytest.approx(63.0) - assert result.sum == pytest.approx(126.0) - - @pytest.mark.asyncio - async def test_process_record_list_metric_summarizes_distribution( - self, mock_run - ) -> None: - accumulator = MetricsAccumulator(mock_run) - message = create_metric_records_message( - x_request_id="test-1", - results=[ - { - InterChunkLatencyMetric.tag: [ - 10_000_000.0, - 20_000_000.0, - 30_000_000.0, - ] - } - ], - ) - - await accumulator.process_record(message.to_data()) - - summary = await accumulator.summarize() - result = summary.results[InterChunkLatencyMetric.tag] - assert result.count == 3 - assert result.sum == pytest.approx(60.0) - assert result.min == pytest.approx(10.0) - assert result.max == pytest.approx(30.0) - - @pytest.mark.asyncio - async def test_process_aggregate_metric_sums_values(self, mock_run) -> None: - accumulator = MetricsAccumulator(mock_run) - - for idx, value in enumerate([5, 3]): - message = create_metric_records_message( - x_request_id=f"test-{idx}", - request_start_ns=1_000_000_000 + idx, - results=[{RequestCountMetric.tag: value}], - ) - await accumulator.process_record(message.to_data()) - - summary = await accumulator.summarize() - - assert summary.results[RequestCountMetric.tag].avg == 8 - - @pytest.mark.asyncio - async def test_derived_metrics_are_computed_from_accumulated_results( - self, mock_run - ) -> None: - accumulator = MetricsAccumulator(mock_run) - message = create_metric_records_message( - x_request_id="test-1", - results=[ - { - RequestCountMetric.tag: 100, - MinRequestTimestampMetric.tag: 0, - MaxResponseTimestampMetric.tag: 10 * NANOS_PER_SECOND, - } - ], - ) - - await accumulator.process_record(message.to_data()) - summary = await accumulator.summarize() - - assert summary.results[RequestThroughputMetric.tag].avg == pytest.approx(10.0) - - @pytest.mark.asyncio - async def test_derived_metrics_ignore_no_metric_value(self, mock_run) -> None: - accumulator = MetricsAccumulator(mock_run) - - def failing_derive_func(results_dict: MetricResultsDict) -> float: - raise NoMetricValue("Cannot derive value") - - accumulator._derive_funcs = {RequestThroughputMetric.tag: failing_derive_func} - - with patch.object(accumulator, "debug") as mock_debug: - summary = await accumulator.summarize() - - assert RequestThroughputMetric.tag not in summary.results - assert any( - "No metric value for derived metric" in str(call.args[0]) - for call in mock_debug.call_args_list - ) - - @pytest.mark.asyncio - async def test_derived_metrics_warn_on_unexpected_exception(self, mock_run) -> None: - accumulator = MetricsAccumulator(mock_run) - - def failing_derive_func(results_dict: MetricResultsDict) -> float: - raise ValueError("Calculation error") - - accumulator._derive_funcs = {RequestThroughputMetric.tag: failing_derive_func} - - with patch.object(accumulator, "warning") as mock_warning: - summary = await accumulator.summarize() - - assert RequestThroughputMetric.tag not in summary.results - mock_warning.assert_called_once() - - @pytest.mark.asyncio - async def test_summarize_returns_typed_summary(self, mock_run) -> None: - accumulator = MetricsAccumulator(mock_run) - await accumulator.process_record( - create_metric_records_message( - x_request_id="test-1", - results=[{RequestLatencyMetric.tag: 42_000_000.0}], - ).to_data() - ) - - summary = await accumulator.summarize() - - assert isinstance(summary, AccumulatorMetricsSummary) - assert summary.results[RequestLatencyMetric.tag].unit == "ms" - assert summary.results[RequestLatencyMetric.tag].avg == pytest.approx(42.0) - diff --git a/tests/unit/post_processors/test_metrics_accumulator.py b/tests/unit/post_processors/test_metrics_accumulator.py index 4be047d8e9..de390ba3ee 100644 --- a/tests/unit/post_processors/test_metrics_accumulator.py +++ b/tests/unit/post_processors/test_metrics_accumulator.py @@ -22,6 +22,9 @@ ) from aiperf.metrics.column_store import ColumnStore from aiperf.metrics.metric_dicts import MetricResultsDict, metric_result_from_array +from aiperf.metrics.types.inter_chunk_latency_metric import InterChunkLatencyMetric +from aiperf.metrics.types.max_response_metric import MaxResponseTimestampMetric +from aiperf.metrics.types.min_request_metric import MinRequestTimestampMetric from aiperf.metrics.types.request_count_metric import RequestCountMetric from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric from aiperf.metrics.types.request_throughput_metric import RequestThroughputMetric @@ -256,6 +259,141 @@ async def test_summarize_realtime_phase_scoped_excludes_warmup( # Request count reflects only the single profiling record, not both. assert summary.results[RequestCountMetric.tag].avg == pytest.approx(1.0) + @pytest.mark.asyncio + async def test_process_record_metric_accumulates_values(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + + for idx, value in enumerate([42_000_000.0, 84_000_000.0]): + message = create_metric_records_message( + x_request_id=f"test-{idx}", + request_start_ns=1_000_000_000 + idx, + request_end_ns=1_100_000_000 + idx, + results=[{RequestLatencyMetric.tag: value}], + ) + await accumulator.process_record(message.to_data()) + + summary = await accumulator.summarize() + result = summary.results[RequestLatencyMetric.tag] + + assert accumulator.record_count == 2 + assert result.unit == "ms" + assert result.count == 2 + assert result.avg == pytest.approx(63.0) + assert result.sum == pytest.approx(126.0) + + @pytest.mark.asyncio + async def test_process_record_list_metric_summarizes_distribution( + self, mock_run + ) -> None: + accumulator = MetricsAccumulator(mock_run) + message = create_metric_records_message( + x_request_id="test-1", + results=[ + { + InterChunkLatencyMetric.tag: [ + 10_000_000.0, + 20_000_000.0, + 30_000_000.0, + ] + } + ], + ) + + await accumulator.process_record(message.to_data()) + + summary = await accumulator.summarize() + result = summary.results[InterChunkLatencyMetric.tag] + assert result.count == 3 + assert result.sum == pytest.approx(60.0) + assert result.min == pytest.approx(10.0) + assert result.max == pytest.approx(30.0) + + @pytest.mark.asyncio + async def test_process_aggregate_metric_sums_values(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + + for idx, value in enumerate([5, 3]): + message = create_metric_records_message( + x_request_id=f"test-{idx}", + request_start_ns=1_000_000_000 + idx, + results=[{RequestCountMetric.tag: value}], + ) + await accumulator.process_record(message.to_data()) + + summary = await accumulator.summarize() + + assert summary.results[RequestCountMetric.tag].avg == 8 + + @pytest.mark.asyncio + async def test_derived_metrics_are_computed_from_accumulated_results( + self, mock_run + ) -> None: + accumulator = MetricsAccumulator(mock_run) + message = create_metric_records_message( + x_request_id="test-1", + results=[ + { + RequestCountMetric.tag: 100, + MinRequestTimestampMetric.tag: 0, + MaxResponseTimestampMetric.tag: 10 * NANOS_PER_SECOND, + } + ], + ) + + await accumulator.process_record(message.to_data()) + summary = await accumulator.summarize() + + assert summary.results[RequestThroughputMetric.tag].avg == pytest.approx(10.0) + + @pytest.mark.asyncio + async def test_derived_metrics_ignore_no_metric_value(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + + def failing_derive_func(results_dict: MetricResultsDict) -> float: + raise NoMetricValue("Cannot derive value") + + accumulator._derive_funcs = {RequestThroughputMetric.tag: failing_derive_func} + + with patch.object(accumulator, "debug") as mock_debug: + summary = await accumulator.summarize() + + assert RequestThroughputMetric.tag not in summary.results + assert any( + "No metric value for derived metric" in str(call.args[0]) + for call in mock_debug.call_args_list + ) + + @pytest.mark.asyncio + async def test_derived_metrics_warn_on_unexpected_exception(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + + def failing_derive_func(results_dict: MetricResultsDict) -> float: + raise ValueError("Calculation error") + + accumulator._derive_funcs = {RequestThroughputMetric.tag: failing_derive_func} + + with patch.object(accumulator, "warning") as mock_warning: + summary = await accumulator.summarize() + + assert RequestThroughputMetric.tag not in summary.results + mock_warning.assert_called_once() + + @pytest.mark.asyncio + async def test_summarize_returns_typed_summary(self, mock_run) -> None: + accumulator = MetricsAccumulator(mock_run) + await accumulator.process_record( + create_metric_records_message( + x_request_id="test-1", + results=[{RequestLatencyMetric.tag: 42_000_000.0}], + ).to_data() + ) + + summary = await accumulator.summarize() + + assert isinstance(summary, AccumulatorMetricsSummary) + assert summary.results[RequestLatencyMetric.tag].unit == "ms" + assert summary.results[RequestLatencyMetric.tag].avg == pytest.approx(42.0) + class TestComputeResultsWindowBounds: """Test that _compute_results propagates window bounds to derived metrics.""" From 8d752e769afc5c0a978b2dbcaadb3da404fcefbc Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 19:14:32 -0700 Subject: [PATCH 09/39] feat(metrics): unified accumulator protocol, first-class analyzer role, energy-efficiency analyzer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finishes three connected pieces of the accumulator architecture (design corpus analysis/03,06 + doc 0005), turning the dead SummaryContext scaffolding into a working cross-accumulator path. #3 — Conform GPU-telemetry & server-metrics accumulators to the unified AccumulatorProtocol: - Both now take summarize(ctx: SummaryContext|None) and export_results(ctx: ExportContext) instead of bespoke positional signatures; ExportContext gains optional warmup_start_ns/warmup_end_ns (server metrics' two-window export). - GPUTelemetryAccumulator exposes total_energy_joules()/total_power_watts() windowed query methods for analyzers. - SummaryContext is now populated (accumulator instances + summaries) and its false "topological-sort pipeline" docstring is corrected to the real accumulators->analyzers flow. #4 — Make `analyzer` a first-class plugin category: - New AnalyzerProtocol + AnalyzerMetadata (dependencies declared by kind: required_accumulators = live instance, required_summaries = summary output). - categories.yaml `analyzer` + regenerated enums/overloads/schemas (AnalyzerType, PluginType.ANALYZER); load_analyzers() loader + RecordsManager._run_analyzers() runs them after accumulators, dependency-gated, and merges their MetricResults. A metadata typo (unknown accumulator name) now errors loudly instead of silently disabling the analyzer. #5 — EnergyEfficiencyAnalyzer as the first real analyzer: - Joins the live GPU accumulator (energy/power) with the metrics summary (tokens/throughput/latency) to emit all 8 energy metrics per doc 0005; adds the 6 missing tags (energy_per_output_token, energy_per_request, energy_per_total_token, performance_per_watt, energy_delay_product, average_gpu_power) + units. - Replaces the inline _apply_gpu_efficiency_metrics/compute_efficiency_metrics hack that string-scanned records and read concurrency off run.cfg. Verified: full unit suite green; new analyzer formula tests (pinned to doc 0005); live E2E against the mock DCGM faker emitted 9/10 energy metrics with exact cross-checks (energy_per_user correctly omitted on a rate-only run). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/cli-options.md | 2 +- docs/metrics-reference.md | 90 +++- docs/plugins/plugin-system.md | 3 +- src/aiperf/common/accumulator_protocols.py | 45 +- src/aiperf/common/enums/metric_enums.py | 4 + src/aiperf/gpu_telemetry/accumulator.py | 150 ++----- .../metrics/energy_efficiency_analyzer.py | 171 ++++++++ .../metrics/types/power_efficiency_metrics.py | 185 +++++---- src/aiperf/plugin/categories.yaml | 11 + src/aiperf/plugin/enums.py | 6 +- src/aiperf/plugin/plugins.py | 8 +- src/aiperf/plugin/plugins.yaml | 18 + src/aiperf/plugin/schema/plugins.schema.json | 58 +++ src/aiperf/plugin/schema/schemas.py | 41 ++ src/aiperf/records/records_manager.py | 129 +++--- .../records/records_manager_processing.py | 71 ++++ src/aiperf/server_metrics/accumulator.py | 31 +- tests/unit/gpu_telemetry/test_accumulator.py | 392 ------------------ .../test_energy_efficiency_analyzer.py | 127 ++++++ .../metrics/test_power_efficiency_metrics.py | 44 +- tests/unit/records/test_records_manager.py | 141 +------ .../test_records_manager_process_results.py | 5 +- tests/unit/server_metrics/test_accumulator.py | 81 ++-- .../test_accumulator_sglang_residuals.py | 7 +- .../server_metrics/test_unknown_metrics.py | 3 +- 25 files changed, 995 insertions(+), 828 deletions(-) create mode 100644 src/aiperf/metrics/energy_efficiency_analyzer.py create mode 100644 tests/unit/metrics/test_energy_efficiency_analyzer.py diff --git a/docs/cli-options.md b/docs/cli-options.md index f3de59ee92..24700c3c12 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -1675,7 +1675,7 @@ Explore AIPerf plugins: aiperf plugins [category] [type] #### `--category` `` Category to explore. -
_Choices: [`accumulator`, `accuracy_benchmark`, `accuracy_grader`, `api_router`, `arrival_pattern`, `communication`, `communication_client`, `console_exporter`, `convergence_criterion`, `custom_dataset_loader`, `data_exporter`, `dataset_backing_store`, `dataset_client_store`, `dataset_composer`, `dataset_sampler`, `endpoint`, `gpu_telemetry_collector`, `plot`, `public_dataset_loader`, `ramp`, `record_processor`, `search_planner`, `search_recipe`, `search_recipe_post_process`, `service`, `service_manager`, `stream_exporter`, `timing_strategy`, `transport`, `ui`, `url_selection_strategy`, `zmq_proxy`]_ +
_Choices: [`accumulator`, `accuracy_benchmark`, `accuracy_grader`, `analyzer`, `api_router`, `arrival_pattern`, `communication`, `communication_client`, `console_exporter`, `convergence_criterion`, `custom_dataset_loader`, `data_exporter`, `dataset_backing_store`, `dataset_client_store`, `dataset_composer`, `dataset_sampler`, `endpoint`, `gpu_telemetry_collector`, `plot`, `public_dataset_loader`, `ramp`, `record_processor`, `search_planner`, `search_recipe`, `search_recipe_post_process`, `service`, `service_manager`, `stream_exporter`, `timing_strategy`, `transport`, `ui`, `url_selection_strategy`, `zmq_proxy`]_ #### `--name` `` diff --git a/docs/metrics-reference.md b/docs/metrics-reference.md index e32a97cd4d..f316c5eac0 100644 --- a/docs/metrics-reference.md +++ b/docs/metrics-reference.md @@ -124,6 +124,12 @@ This document provides a comprehensive reference of all metrics available in AIP - [Total GPU Energy](#total-gpu-energy) - [Output Tokens per Joule](#output-tokens-per-joule) - [Energy per User](#energy-per-user) + - [Average GPU Power](#average-gpu-power) + - [Energy per Output Token](#energy-per-output-token) + - [Energy per Total Token](#energy-per-total-token) + - [Energy per Request](#energy-per-request) + - [Performance per Watt](#performance-per-watt) + - [Energy Delay Product](#energy-delay-product) - [Metric Flags Reference](#metric-flags-reference) --- @@ -1870,9 +1876,9 @@ http_req_chunks_received = trace.response_chunks_count ## GPU Power Efficiency Metrics > [!NOTE] -> All metrics in this section require `--gpu-telemetry` to be enabled and the underlying collector (DCGM, pynvml, or amdsmi) to expose the relevant signal (`gpu_power_usage` and/or `energy_consumption`). They are computed once per profiling phase by `GPUTelemetryAccumulator.compute_efficiency_metrics`, not by the standard derivation walk — see the [Externally-Injected Derived Metric pattern](dev/patterns.md#externally-injected-derived-metric-pattern). +> All metrics in this section require `--gpu-telemetry` to be enabled and the underlying collector (DCGM, pynvml, or amdsmi) to expose the relevant signal (`gpu_power_usage` and/or `energy_consumption`). They are computed once per profiling phase by the `EnergyEfficiencyAnalyzer` — an `analyzer` plugin that joins the GPU-telemetry accumulator (energy/power) to the metrics-accumulator summary (tokens/throughput/latency) via the `SummaryContext` — not by the standard derivation walk. See the [Externally-Injected Derived Metric pattern](dev/patterns.md#externally-injected-derived-metric-pattern) and the `analyzer` plugin category. -Each metric's header surfaces the number of GPUs that contributed valid data (e.g. `Total GPU Power (8 GPUs)`), so a partial-cohort run (where one or more GPUs failed to report) is distinguishable from a full run. Tags are emitted in this order when present: `total_gpu_power`, `total_gpu_energy`, `output_tokens_per_joule`, `energy_per_user`. Each tag is independently omitted when its underlying signal is unavailable. +The family is skipped entirely when GPU telemetry is not collected. Each tag is independently omitted when its underlying signal is unavailable (e.g. `energy_per_user` only appears for concurrency runs). Tags: `energy_delay_product`, `performance_per_watt`, `average_gpu_power`, `total_gpu_energy`, `total_gpu_power`, `energy_per_total_token`, `energy_per_output_token`, `energy_per_request`, `output_tokens_per_joule`, `energy_per_user`. ### Total GPU Power @@ -1961,11 +1967,89 @@ energy_per_user_j = total_gpu_energy / concurrency - Unit: `joules/user`. - Flagged `MetricFlags.NONE` — smaller-is-better is the default for unflagged metrics. - Denominator is the profiling phase's configured `concurrency`. The resolver defaults this to `1` when `--concurrency` isn't specified in concurrency-mode runs, so the metric is emitted in the common case. -- Header reports the energy-side GPU count (the same cohort `total_gpu_energy` reports), e.g. `Energy per User (8 GPUs)`. +- Denominator is the profiling phase's configured `concurrency` (`run.cfg.get_profiling_phases()[0].concurrency`). - Omitted when concurrency is unset (e.g. pure `--request-rate` mode) or aggregate GPU energy is unavailable. --- +### Average GPU Power + +**Type:** [Derived Metric](#derived-metrics) (externally injected) + +Mean GPU power draw per reporting GPU over the profiling window, in watts (`W`). Complements `total_gpu_power` (the fleet sum) with a per-GPU view. + +**Formula:** +```python +average_gpu_power_w = total_gpu_power / reporting_gpu_count +``` + +--- + +### Energy per Output Token + +**Type:** [Derived Metric](#derived-metrics) (externally injected) + +The primary energy-efficiency metric: GPU energy spent per output token, in millijoules per token (`mJ/token`). Lower is better. Normalizes across model sizes, hardware, and batch sizes. + +**Formula:** +```python +energy_per_output_token_mj = total_gpu_energy * 1000 / total_output_tokens +``` + +--- + +### Energy per Total Token + +**Type:** [Derived Metric](#derived-metrics) (externally injected) + +GPU energy per total (input + output) token, in `mJ/token`. Captures combined prefill + decode cost; useful when comparing systems with different input/output ratios. + +**Formula:** +```python +energy_per_total_token_mj = total_gpu_energy * 1000 / (total_isl + total_output_tokens) +``` + +--- + +### Energy per Request + +**Type:** [Derived Metric](#derived-metrics) (externally injected) + +GPU energy consumed per request, in joules per request (`joules/request`). + +**Formula:** +```python +energy_per_request_j = total_gpu_energy / request_count +``` + +--- + +### Performance per Watt + +**Type:** [Derived Metric](#derived-metrics) (externally injected) + +Request throughput per watt of GPU power draw, in `requests/sec/W`. Higher is better — the standard HPC/MLPerf efficiency metric, enabling apples-to-apples comparison across GPU SKUs and power envelopes. + +**Formula:** +```python +performance_per_watt = request_throughput / total_gpu_power +``` + +--- + +### Energy Delay Product + +**Type:** [Derived Metric](#derived-metrics) (externally injected) + +Joint energy-and-latency figure of merit, in joule-seconds (`J*s`). Lower is better — penalizes both slow-but-efficient and fast-but-wasteful configurations, useful for finding the Pareto-optimal operating point. + +**Formula:** +```python +energy_delay_product = energy_per_request * mean_request_latency_s +``` + +--- + ## Multi-Run Aggregate Metrics > [!NOTE] diff --git a/docs/plugins/plugin-system.md b/docs/plugins/plugin-system.md index d8151ee623..fd68ae8ce5 100644 --- a/docs/plugins/plugin-system.md +++ b/docs/plugins/plugin-system.md @@ -100,7 +100,7 @@ for entry, cls in plugins.iter_all(PluginType.ENDPOINT): ## Plugin Categories -AIPerf supports 32 plugin categories organized by function, including `api_router` and `public_dataset_loader`: +AIPerf supports 33 plugin categories organized by function, including `api_router` and `public_dataset_loader`: ### Timing Categories @@ -135,6 +135,7 @@ AIPerf supports 32 plugin categories organized by function, including `api_route |----------|------|-------------| | `record_processor` | `RecordProcessorType` | Per-record metric computation | | `accumulator` | `AccumulatorType` | Record-type-routed aggregation and summary computation | +| `analyzer` | `AnalyzerType` | Summarize-time cross-accumulator joins (e.g. energy efficiency), reading peers via `SummaryContext` | | `stream_exporter` | `StreamExporterType` | Record-type-routed streaming sinks such as JSONL and OpenTelemetry | | `data_exporter` | `DataExporterType` | File format exporters (CSV, JSON, Parquet) | | `console_exporter` | `ConsoleExporterType` | Terminal output exporters | diff --git a/src/aiperf/common/accumulator_protocols.py b/src/aiperf/common/accumulator_protocols.py index 980b43f094..e4a7c421ed 100644 --- a/src/aiperf/common/accumulator_protocols.py +++ b/src/aiperf/common/accumulator_protocols.py @@ -79,14 +79,28 @@ class ExportContext: cancelled: bool = False """True when the profile run was cancelled — exporters may emit partial artifacts.""" + warmup_start_ns: int | None = None + """Inclusive start of the warmup window (ns), for accumulators that export a + separate warmup summary alongside the profiling one (e.g. server metrics).""" + + warmup_end_ns: int | None = None + """Exclusive end of the warmup window (ns); see ``warmup_start_ns``.""" + @dataclass(slots=True) class SummaryContext: - """Typed cross-accumulator communication context for dependency-ordered summarization. - - NOT a Pydantic model — this is never serialized over the wire. It is created - by RecordsManager._process_results() and passed through the topological-sort - pipeline so each accumulator can read outputs from its declared dependencies. + """Typed cross-accumulator communication context for summarize-time analyzers. + + NOT a Pydantic model — never serialized over the wire. Created by + RecordsManager during summarization: accumulators run first and register + their instances in ``accumulators`` and their summaries in + ``accumulator_outputs``; then ``analyzer`` plugins read peer state via + ``get_accumulator()`` / ``get_output()`` to compute cross-accumulator + metrics (e.g. energy efficiency joins GPU telemetry to inference tokens). + + Dependencies are currently one level deep (analyzers depend on accumulators, + not on each other), so a flat two-stage run suffices; a topological sort + over analyzer-to-analyzer dependencies is the extension point if that changes. """ accumulators: dict[AccumulatorType, Any] = field(default_factory=dict) @@ -159,6 +173,27 @@ async def export_results(self, ctx: ExportContext) -> Any: ... +@runtime_checkable +class AnalyzerProtocol(Protocol): + """Protocol for summarize-time analyzers that join across accumulators. + + Analyzers run once after every accumulator has summarized. Unlike an + accumulator, an analyzer stores no records — it reads peer accumulator state + from the ``SummaryContext`` (via ``get_accumulator()`` / ``get_output()``) + and returns derived ``MetricResult`` rows that are merged into the profiling + summary. The first analyzer is energy efficiency, which joins GPU-telemetry + energy to inference token totals. + + An analyzer declares its ``required_accumulators`` in plugin metadata; the + RecordsManager skips it when any declared accumulator is absent (e.g. energy + efficiency is skipped when GPU telemetry is disabled). + """ + + async def analyze(self, ctx: SummaryContext) -> list[MetricResult]: + """Compute cross-accumulator metrics from ``ctx`` and return them.""" + ... + + @runtime_checkable class StreamExporterProtocol(Protocol): """Protocol for processors that stream each record to an external sink (e.g. JSONL files). diff --git a/src/aiperf/common/enums/metric_enums.py b/src/aiperf/common/enums/metric_enums.py index 3df836dca0..3d34f1cc3e 100644 --- a/src/aiperf/common/enums/metric_enums.py +++ b/src/aiperf/common/enums/metric_enums.py @@ -190,10 +190,14 @@ class GenericMetricUnit(BaseMetricUnit): ERRORS = _unit("errors") IMAGE = _unit("image") IMAGES = _unit("images") + JOULES_PER_REQUEST = _unit("joules/request") JOULES_PER_USER = _unit("joules/user") + JOULE_SECONDS = _unit("J*s") + MILLIJOULES_PER_TOKEN = _unit("mJ/token") PERCENT = _unit("%") RATIO = _unit("ratio") REQUESTS = _unit("requests") + REQUESTS_PER_SECOND_PER_WATT = _unit("requests/sec/W") TOKENS = _unit("tokens") TOKENS_PER_JOULE = _unit("tokens/J") USER = _unit("user") diff --git a/src/aiperf/gpu_telemetry/accumulator.py b/src/aiperf/gpu_telemetry/accumulator.py index b9d941d52c..737dcf474e 100644 --- a/src/aiperf/gpu_telemetry/accumulator.py +++ b/src/aiperf/gpu_telemetry/accumulator.py @@ -11,7 +11,6 @@ from aiperf.common.constants import NANOS_PER_SECOND from aiperf.common.enums import ( EnergyMetricUnit, - GenericMetricUnit, GPUTelemetryMode, PowerMetricUnit, ) @@ -22,7 +21,6 @@ from aiperf.common.messages import RealtimeTelemetryMetricsMessage from aiperf.common.models import ( EndpointData, - ErrorDetailsCount, GpuSummary, MetricResult, TelemetryExportData, @@ -40,14 +38,10 @@ from aiperf.post_processors.base_metrics_processor import BaseMetricsProcessor if TYPE_CHECKING: + from aiperf.common.accumulator_protocols import ExportContext, SummaryContext from aiperf.config.resolution.plan import BenchmarkRun -def _gpu_count_suffix(n: int) -> str: - """Render a "(N GPUs)" header suffix; partial-cohort runs differ from full.""" - return f"({n} GPU{'s' if n != 1 else ''})" - - class GPUTelemetryAccumulator(BaseMetricsProcessor): """Accumulate GPU telemetry records and compute metrics in a hierarchical structure. @@ -182,7 +176,9 @@ async def _report_realtime_metrics(self) -> None: ) self._last_metric_values = new_values - async def summarize(self) -> list[MetricResult]: + async def summarize( + self, ctx: "SummaryContext | None" = None + ) -> list[MetricResult]: """Generate per-GPU MetricResult list for real-time display and final export. This method is called by RecordsManager for: @@ -240,11 +236,8 @@ async def summarize(self) -> list[MetricResult]: return results - def export_results( - self, - start_ns: int | None = None, - end_ns: int | None = None, - error_summary: list[ErrorDetailsCount] | None = None, + async def export_results( + self, ctx: "ExportContext" ) -> "TelemetryExportData | None": """Export accumulated telemetry data as a TelemetryExportData object. @@ -256,15 +249,16 @@ def export_results( - Counter metrics (energy, errors): Delta computed from baseline before start_ns Args: - start_ns: Start time of profiling phase in nanoseconds (excludes warmup). - If None, includes all data from beginning. - end_ns: End time of profiling phase in nanoseconds. If None, includes all - data after start_ns (including final scrape after profiling completes). - error_summary: Optional list of error counts + ctx: ExportContext with ``start_ns`` (profiling start, excludes warmup; + None = from beginning), ``end_ns`` (None = through the final scrape), + and ``error_summary``. Returns: TelemetryExportData object with pre-computed metrics for each GPU """ + start_ns = ctx.start_ns + end_ns = ctx.end_ns + error_summary = ctx.error_summary # Create time filter for warmup exclusion # Note: end_ns is typically None to include the final telemetry scrape # that occurs after PROFILE_COMPLETE but before export @@ -435,106 +429,34 @@ def _sum_gpu_energy_joules(self, time_filter: TimeRangeFilter) -> tuple[float, i gpu_count += 1 return total_energy_j, gpu_count - def compute_efficiency_metrics( - self, - metric_results: list[MetricResult], - time_filter: TimeRangeFilter, - ) -> list[MetricResult]: - """Compute cross-boundary power efficiency totals for one profiling phase. + def total_power_watts( + self, start_ns: int | None, end_ns: int | None + ) -> tuple[float, int]: + """Cross-GPU total of avg(gpu_power_usage) over ``[start_ns, end_ns)``. - Aggregates avg(gpu_power_usage) and energy_consumption deltas across - all GPUs into 0-4 cross-GPU totals (W, J, tokens/J, J/user). Sync; - called once per phase from `RecordsManager._summarize_with_logging`. - Sibling `summarize()` above is async, periodic for the dashboard, and - emits one MetricResult per GPU per signal instead. - - Args: - metric_results: Read-only; scanned only for the `total_output_tokens` - tag to compute tokens/J. - time_filter: Profiling-phase window (warmup excluded). For energy, - `end_ns` is widened by `Environment.GPU.FINAL_SCRAPE_GRACE_NS` - to capture the trailing scrape without leaking idle - or subsequent-phase samples. + Query surface for cross-accumulator analyzers (e.g. energy efficiency). + Returns ``(total_power_watts, gpu_count)``; ``gpu_count == 0`` means no + power signal was available. + """ + return self._sum_gpu_power_watts( + TimeRangeFilter(start_ns=start_ns, end_ns=end_ns) + ) - Returns: - Up to 4 MetricResults: `total_gpu_power`, `total_gpu_energy`, - `output_tokens_per_joule`, `energy_per_user`. Each is independently - omitted when its underlying signal is missing. + def total_energy_joules( + self, start_ns: int | None, end_ns: int | None + ) -> tuple[float, int]: + """Cross-GPU total energy (J) over ``[start_ns, end_ns)``. - Example: - >>> accumulator.compute_efficiency_metrics( - ... records, TimeRangeFilter(start_ns=t0, end_ns=t1)) + Query surface for cross-accumulator analyzers. ``end_ns`` is widened by + ``Environment.GPU.FINAL_SCRAPE_GRACE_NS`` so the trailing scrape that + closes the phase is captured (see ``_sum_gpu_energy_joules``). Returns + ``(total_energy_joules, gpu_count)``. """ - tokens_result = next( - (r for r in metric_results if r.tag == "total_output_tokens"), None - ) - total_output_tokens = tokens_result.avg if tokens_result is not None else None - total_power_w, power_count = self._sum_gpu_power_watts(time_filter) - bounded_end_ns = time_filter.end_ns + Environment.GPU.FINAL_SCRAPE_GRACE_NS - energy_filter = TimeRangeFilter(start_ns=time_filter.start_ns, end_ns=bounded_end_ns) # fmt: skip - total_energy_j, energy_count = self._sum_gpu_energy_joules(energy_filter) - profiling_phases = self.run.cfg.get_profiling_phases() - raw_concurrency = profiling_phases[0].concurrency if profiling_phases else None - concurrency = ( - raw_concurrency - if isinstance(raw_concurrency, int) - and not isinstance(raw_concurrency, bool) - and raw_concurrency > 0 + bounded_end_ns = ( + end_ns + Environment.GPU.FINAL_SCRAPE_GRACE_NS + if end_ns is not None else None ) - self.debug( - lambda: ( - f"compute_efficiency_metrics totals: " - f"power={total_power_w:.2f}W ({power_count} GPUs), " - f"energy={total_energy_j:.2f}J ({energy_count} GPUs), " - f"total_output_tokens={total_output_tokens}, " - f"concurrency={concurrency}" - ) + return self._sum_gpu_energy_joules( + TimeRangeFilter(start_ns=start_ns, end_ns=bounded_end_ns) ) - - results: list[MetricResult] = [] - if power_count > 0: - results.append(MetricResult( - tag="total_gpu_power", header=f"Total GPU Power {_gpu_count_suffix(power_count)}", - unit=str(PowerMetricUnit.WATT), avg=total_power_w, count=None, - )) # fmt: skip - else: - self.debug("No GPU power data available") - - if energy_count > 0: - results.append(MetricResult( - tag="total_gpu_energy", header=f"Total GPU Energy {_gpu_count_suffix(energy_count)}", - unit=str(EnergyMetricUnit.JOULE), avg=total_energy_j, count=None, - )) # fmt: skip - else: - self.debug("No GPU energy data available, skipping total_gpu_energy") - - if total_output_tokens is not None and total_energy_j > 0: - results.append(MetricResult( - tag="output_tokens_per_joule", header=f"Output Tokens per Joule {_gpu_count_suffix(energy_count)}", - unit=str(GenericMetricUnit.TOKENS_PER_JOULE), - avg=total_output_tokens / total_energy_j, count=None, - )) # fmt: skip - else: - self.debug( - lambda: ( - f"Skipping output_tokens_per_joule: " - f"total_output_tokens={total_output_tokens}, " - f"total_energy_j={total_energy_j:.2f}" - ) - ) - - if concurrency is not None and energy_count > 0: - results.append(MetricResult( - tag="energy_per_user", header=f"Energy per User {_gpu_count_suffix(energy_count)}", - unit=str(GenericMetricUnit.JOULES_PER_USER), - avg=total_energy_j / concurrency, count=None, - )) # fmt: skip - else: - self.debug( - lambda: ( - f"Skipping energy_per_user: " - f"concurrency={concurrency}, energy_count={energy_count}" - ) - ) - return results diff --git a/src/aiperf/metrics/energy_efficiency_analyzer.py b/src/aiperf/metrics/energy_efficiency_analyzer.py new file mode 100644 index 0000000000..7bccb76521 --- /dev/null +++ b/src/aiperf/metrics/energy_efficiency_analyzer.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Energy-efficiency analyzer: the first cross-accumulator ``analyzer`` plugin. + +Joins GPU-telemetry energy/power (a live query on the ``GPUTelemetryAccumulator`` +over the profiling window) with inference token/throughput/latency totals (read +off the ``MetricsAccumulator`` summary) to emit the energy-efficiency metric +family. Runs at summarize time via the SummaryContext; skipped by RecordsManager +when GPU telemetry is not collected. See design doc ``0005-energy-efficiency-metrics.md``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from aiperf.common.logging import AIPerfLogger +from aiperf.common.models import MetricResult +from aiperf.metrics.types.power_efficiency_metrics import ( + AverageGpuPowerMetric, + EnergyDelayProductMetric, + EnergyPerOutputTokenMetric, + EnergyPerRequestMetric, + EnergyPerTotalTokenMetric, + EnergyPerUserMetric, + OutputTokensPerJouleMetric, + PerformancePerWattMetric, + TotalGpuEnergyMetric, + TotalGpuPowerMetric, +) +from aiperf.plugin.enums import AccumulatorType + +if TYPE_CHECKING: + from collections.abc import Callable + + from aiperf.common.accumulator_protocols import SummaryContext + from aiperf.config.resolution.plan import BenchmarkRun + +_logger = AIPerfLogger(__name__) + +_MS_PER_SECOND = 1000.0 + + +def _result(metric_cls: type, value: float) -> MetricResult: + """Build the injected MetricResult for an energy metric class.""" + return MetricResult( + tag=metric_cls.tag, + header=metric_cls.header, + unit=str(metric_cls.unit), + avg=value, + count=None, + ) + + +class EnergyEfficiencyAnalyzer: + """Summarize-time analyzer emitting GPU energy-efficiency metrics. + + Reads the live ``GPUTelemetryAccumulator`` (windowed energy/power) and the + ``MetricsAccumulator`` summary (token/throughput/latency totals) from the + SummaryContext, then computes the energy metric family. Each metric is + emitted only when its inputs are available, mirroring the per-signal omission + of the pipeline it replaces. + """ + + def __init__( + self, + service_id: str | None = None, + run: BenchmarkRun | None = None, + pub_client: Any = None, + **kwargs: Any, + ) -> None: + self.run = run + + def _concurrency(self) -> int | None: + """Positive integer profiling concurrency, or None (rate-only runs).""" + if self.run is None: + return None + phases = self.run.cfg.get_profiling_phases() + raw = phases[0].concurrency if phases else None + if isinstance(raw, int) and not isinstance(raw, bool) and raw > 0: + return raw + return None + + async def analyze(self, ctx: SummaryContext) -> list[MetricResult]: + gpu = ctx.get_accumulator(AccumulatorType.GPU_TELEMETRY) + summary = ctx.get_output(AccumulatorType.METRIC_RESULTS) + if gpu is None or summary is None: + return [] + results_by_tag = getattr(summary, "results", None) + if not isinstance(results_by_tag, dict): + return [] + + def metric(tag: str) -> float | None: + r = results_by_tag.get(tag) + return r.avg if r is not None and r.avg is not None else None + + start_ns = ctx.start_ns or None + end_ns = ctx.end_ns or None + energy_j, energy_count = gpu.total_energy_joules(start_ns, end_ns) + power_w, power_count = gpu.total_power_watts(start_ns, end_ns) + + out = self._power_metrics(power_w, power_count) + out += self._energy_metrics(energy_j, energy_count, metric) + if ( + power_count > 0 + and power_w > 0 + and (throughput := metric("request_throughput")) + ): + out.append(_result(PerformancePerWattMetric, throughput / power_w)) + + _logger.debug( + lambda: ( + f"EnergyEfficiencyAnalyzer emitted {len(out)} metrics " + f"(energy={energy_j:.2f}J/{energy_count}gpu, " + f"power={power_w:.2f}W/{power_count}gpu)" + ) + ) + return out + + @staticmethod + def _power_metrics(power_w: float, power_count: int) -> list[MetricResult]: + if power_count <= 0: + return [] + return [ + _result(TotalGpuPowerMetric, power_w), + _result(AverageGpuPowerMetric, power_w / power_count), + ] + + def _energy_metrics( + self, + energy_j: float, + energy_count: int, + metric: Callable[[str], float | None], + ) -> list[MetricResult]: + if energy_count <= 0 or energy_j <= 0: + return [] + out: list[MetricResult] = [_result(TotalGpuEnergyMetric, energy_j)] + + output_tokens = metric("total_output_tokens") + if output_tokens: + out.append(_result(OutputTokensPerJouleMetric, output_tokens / energy_j)) + out.append( + _result( + EnergyPerOutputTokenMetric, + energy_j * _MS_PER_SECOND / output_tokens, + ) + ) + total_tokens = (metric("total_isl") or 0.0) + (output_tokens or 0.0) + if total_tokens > 0: + out.append( + _result( + EnergyPerTotalTokenMetric, energy_j * _MS_PER_SECOND / total_tokens + ) + ) + + energy_per_request_j = None + if request_count := metric("request_count"): + energy_per_request_j = energy_j / request_count + out.append(_result(EnergyPerRequestMetric, energy_per_request_j)) + if (concurrency := self._concurrency()) is not None: + out.append(_result(EnergyPerUserMetric, energy_j / concurrency)) + + # Energy-delay product: J/request * mean request latency (s). + latency_ms = metric("request_latency") + if energy_per_request_j is not None and latency_ms: + out.append( + _result( + EnergyDelayProductMetric, + energy_per_request_j * (latency_ms / _MS_PER_SECOND), + ) + ) + return out diff --git a/src/aiperf/metrics/types/power_efficiency_metrics.py b/src/aiperf/metrics/types/power_efficiency_metrics.py index e970327796..6d22c494cf 100644 --- a/src/aiperf/metrics/types/power_efficiency_metrics.py +++ b/src/aiperf/metrics/types/power_efficiency_metrics.py @@ -1,5 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +"""Registration classes for the GPU energy-efficiency metric family. + +These are ``BaseDerivedMetric`` placeholders: they register each tag's +header/unit/display-order/flags so the summary and exporters recognize them, but +their values are injected post-aggregation by +:class:`aiperf.metrics.energy_efficiency_analyzer.EnergyEfficiencyAnalyzer`, +which joins GPU-telemetry energy/power to inference token totals across +accumulators. ``_derive_value`` therefore always defers with ``NoMetricValue`` so +``MetricsAccumulator._resolve_derived_metrics`` skips them during its derivation +walk; the analyzer supplies the real values. + +See design doc ``0005-energy-efficiency-metrics.md``. +""" from typing import NoReturn @@ -14,109 +27,135 @@ from aiperf.metrics.metric_dicts import MetricResultsDict -class TotalGpuPowerMetric(BaseDerivedMetric[float]): - """Sum of average GPU power across all GPUs during the benchmark phase, in watts. +class _InjectedEnergyMetric(BaseDerivedMetric[float]): + """Shared deferred-derivation base for the analyzer-injected energy metrics. - Invariant: externally injected by - `GPUTelemetryAccumulator.compute_efficiency_metrics` from gpu_power_usage - scrapes. `_derive_value` is intentionally non-functional; - `MetricsAccumulator._resolve_derived_metrics` is expected to catch - NoMetricValue and skip the tag during its derivation walk. + Not registered itself (abstract). The energy-efficiency metrics cannot be + derived from a single ``MetricResultsDict`` because they need GPU telemetry + that lives in a different accumulator; ``EnergyEfficiencyAnalyzer`` computes + them at summarize time via the SummaryContext and injects the results. """ - tag = "total_gpu_power" - header = "Total GPU Power" - unit = PowerMetricUnit.WATT - display_order = 900 - flags = MetricFlags.NONE + __is_abstract__ = True def _derive_value(self, metric_results: MetricResultsDict) -> NoReturn: raise NoMetricValue( - "Cannot derive 'total_gpu_power' from MetricResultsDict: this metric " - "is externally injected by " - "GPUTelemetryAccumulator.compute_efficiency_metrics. If this exception " - "surfaces, the derivation walk is missing its NoMetricValue handler " - "(see MetricsAccumulator._resolve_derived_metrics)." + f"'{self.tag}' is injected post-aggregation by EnergyEfficiencyAnalyzer " + "from GPU telemetry + inference token totals; it cannot be derived from " + "a single MetricResultsDict. If this surfaces, the derivation walk is " + "missing its NoMetricValue handler (MetricsAccumulator." + "_resolve_derived_metrics)." ) -class TotalGpuEnergyMetric(BaseDerivedMetric[float]): - """Sum of GPU energy consumed across all GPUs during the benchmark phase, in joules. +class EnergyDelayProductMetric(_InjectedEnergyMetric): + """Energy-delay product: ``energy_per_request_J * mean_request_latency_s`` (J*s).""" + + __is_abstract__ = False + tag = "energy_delay_product" + header = "Energy Delay Product" + unit = GenericMetricUnit.JOULE_SECONDS + display_order = 700 + flags = MetricFlags.NONE + + +class PerformancePerWattMetric(_InjectedEnergyMetric): + """Request throughput per watt of GPU power draw (req/s/W).""" + + __is_abstract__ = False + tag = "performance_per_watt" + header = "Performance per Watt" + unit = GenericMetricUnit.REQUESTS_PER_SECOND_PER_WATT + display_order = 710 + flags = MetricFlags.LARGER_IS_BETTER - Invariant: externally injected by - `GPUTelemetryAccumulator.compute_efficiency_metrics` from - energy_consumption counter deltas. `_derive_value` is intentionally - non-functional; `MetricsAccumulator._resolve_derived_metrics` is - expected to catch NoMetricValue and skip the tag during its - derivation walk. - """ +class AverageGpuPowerMetric(_InjectedEnergyMetric): + """Mean GPU power draw per GPU over the profiling window (W).""" + + __is_abstract__ = False + tag = "average_gpu_power" + header = "Average GPU Power" + unit = PowerMetricUnit.WATT + display_order = 720 + flags = MetricFlags.NONE + + +class TotalGpuEnergyMetric(_InjectedEnergyMetric): + """Sum of GPU energy consumed across all GPUs during the phase (J).""" + + __is_abstract__ = False tag = "total_gpu_energy" header = "Total GPU Energy" unit = EnergyMetricUnit.JOULE - display_order = 901 + display_order = 730 flags = MetricFlags.NONE - def _derive_value(self, metric_results: MetricResultsDict) -> NoReturn: - raise NoMetricValue( - "Cannot derive 'total_gpu_energy' from MetricResultsDict: this metric " - "is externally injected by " - "GPUTelemetryAccumulator.compute_efficiency_metrics. If this exception " - "surfaces, the derivation walk is missing its NoMetricValue handler " - "(see MetricsAccumulator._resolve_derived_metrics)." - ) + +class TotalGpuPowerMetric(_InjectedEnergyMetric): + """Sum of average GPU power across all GPUs during the phase (W).""" + + __is_abstract__ = False + tag = "total_gpu_power" + header = "Total GPU Power" + unit = PowerMetricUnit.WATT + display_order = 740 + flags = MetricFlags.NONE -class OutputTokensPerJouleMetric(BaseDerivedMetric[float]): - """Total output tokens divided by total GPU energy consumed, in tokens per joule. +class EnergyPerTotalTokenMetric(_InjectedEnergyMetric): + """GPU energy per total (input + output) token (mJ/token).""" - Invariant: externally injected by - `GPUTelemetryAccumulator.compute_efficiency_metrics` as - `total_output_tokens / total_gpu_energy`. `_derive_value` is - intentionally non-functional; - `MetricsAccumulator._resolve_derived_metrics` is expected to catch - NoMetricValue and skip the tag during its derivation walk. - """ + __is_abstract__ = False + tag = "energy_per_total_token" + header = "Energy per Total Token" + unit = GenericMetricUnit.MILLIJOULES_PER_TOKEN + display_order = 745 + flags = MetricFlags.NONE + +class EnergyPerOutputTokenMetric(_InjectedEnergyMetric): + """GPU energy per output token — the primary efficiency metric (mJ/token).""" + + __is_abstract__ = False + tag = "energy_per_output_token" + header = "Energy per Output Token" + unit = GenericMetricUnit.MILLIJOULES_PER_TOKEN + display_order = 750 + flags = MetricFlags.PRODUCES_TOKENS_ONLY + + +class EnergyPerRequestMetric(_InjectedEnergyMetric): + """GPU energy per request (J/request).""" + + __is_abstract__ = False + tag = "energy_per_request" + header = "Energy per Request" + unit = GenericMetricUnit.JOULES_PER_REQUEST + display_order = 755 + flags = MetricFlags.NONE + + +class OutputTokensPerJouleMetric(_InjectedEnergyMetric): + """Total output tokens per joule of GPU energy (tokens/J).""" + + __is_abstract__ = False tag = "output_tokens_per_joule" header = "Output Tokens per Joule" unit = GenericMetricUnit.TOKENS_PER_JOULE - display_order = 902 + display_order = 760 flags = MetricFlags.LARGER_IS_BETTER | MetricFlags.PRODUCES_TOKENS_ONLY - def _derive_value(self, metric_results: MetricResultsDict) -> NoReturn: - raise NoMetricValue( - "Cannot derive 'output_tokens_per_joule' from MetricResultsDict: this " - "metric is externally injected by " - "GPUTelemetryAccumulator.compute_efficiency_metrics. If this exception " - "surfaces, the derivation walk is missing its NoMetricValue handler " - "(see MetricsAccumulator._resolve_derived_metrics)." - ) +class EnergyPerUserMetric(_InjectedEnergyMetric): + """Total GPU energy divided by configured concurrency (J/user). -class EnergyPerUserMetric(BaseDerivedMetric[float]): - """Total GPU energy divided by configured concurrency, in joules per user. - - Invariant: externally injected by - `GPUTelemetryAccumulator.compute_efficiency_metrics` as - `total_gpu_energy / profiling_phase.concurrency`. `_derive_value` is - intentionally non-functional; - `MetricsAccumulator._resolve_derived_metrics` is expected to catch - NoMetricValue and skip the tag during its derivation walk. Omitted - when concurrency is unset (e.g. pure request-rate runs) or zero. + Omitted when concurrency is unset (e.g. pure request-rate runs) or zero. """ + __is_abstract__ = False tag = "energy_per_user" header = "Energy per User" unit = GenericMetricUnit.JOULES_PER_USER - display_order = 903 + display_order = 765 flags = MetricFlags.NONE - - def _derive_value(self, metric_results: MetricResultsDict) -> NoReturn: - raise NoMetricValue( - "Cannot derive 'energy_per_user' from MetricResultsDict: this metric " - "is externally injected by " - "GPUTelemetryAccumulator.compute_efficiency_metrics. If this exception " - "surfaces, the derivation walk is missing its NoMetricValue handler " - "(see MetricsAccumulator._resolve_derived_metrics)." - ) diff --git a/src/aiperf/plugin/categories.yaml b/src/aiperf/plugin/categories.yaml index fc3a15b9d0..536eda4036 100644 --- a/src/aiperf/plugin/categories.yaml +++ b/src/aiperf/plugin/categories.yaml @@ -170,6 +170,17 @@ stream_exporter: arrives. No summarization dependencies. Finalized after all records are processed. One-to-many mapping: multiple exporters loaded simultaneously. +analyzer: + protocol: aiperf.common.accumulator_protocols:AnalyzerProtocol + metadata_class: aiperf.plugin.schema.schemas:AnalyzerMetadata + enum: AnalyzerType + description: | + Analyzers run at summarize time and join across accumulators, reading peer + state from the SummaryContext to compute cross-accumulator metrics (e.g. + energy efficiency joins GPU telemetry to inference tokens). They store no + records and declare the accumulators they require; skipped when any is absent. + One-to-many mapping: multiple analyzers loaded simultaneously. + # ============================================================================= # Accuracy Categories # ============================================================================= diff --git a/src/aiperf/plugin/enums.py b/src/aiperf/plugin/enums.py index f8cbb07577..7dce3a54b0 100644 --- a/src/aiperf/plugin/enums.py +++ b/src/aiperf/plugin/enums.py @@ -12,7 +12,7 @@ from aiperf.plugin import plugins from aiperf.plugin.extensible_enums import create_enum -__all__ = ["APIRouterType", "APIRouterTypeStr", "AccumulatorType", "AccumulatorTypeStr", "AccuracyBenchmarkType", "AccuracyBenchmarkTypeStr", "AccuracyGraderType", "AccuracyGraderTypeStr", "ArrivalPattern", "ArrivalPatternStr", "CommClientType", "CommClientTypeStr", "CommunicationBackend", "CommunicationBackendStr", "ComposerType", "ComposerTypeStr", "ConsoleExporterType", "ConsoleExporterTypeStr", "ConvergenceCriterionType", "ConvergenceCriterionTypeStr", "CustomDatasetType", "CustomDatasetTypeStr", "DataExporterType", "DataExporterTypeStr", "DatasetBackingStoreType", "DatasetBackingStoreTypeStr", "DatasetClientStoreType", "DatasetClientStoreTypeStr", "DatasetFormat", "DatasetFormatStr", "DatasetSamplingStrategy", "DatasetSamplingStrategyStr", "EndpointType", "EndpointTypeStr", "GPUTelemetryCollectorType", "GPUTelemetryCollectorTypeStr", "PhaseType", "PhaseTypeStr", "PlotType", "PlotTypeStr", "PluginType", "PluginTypeStr", "PublicDatasetType", "PublicDatasetTypeStr", "RampType", "RampTypeStr", "RecordProcessorType", "RecordProcessorTypeStr", "SearchPlannerType", "SearchPlannerTypeStr", "SearchRecipePostProcessType", "SearchRecipePostProcessTypeStr", "SearchRecipeType", "SearchRecipeTypeStr", "ServiceRunType", "ServiceRunTypeStr", "ServiceType", "ServiceTypeStr", "StreamExporterType", "StreamExporterTypeStr", "TimingMode", "TimingModeStr", "TransportType", "TransportTypeStr", "UIType", "UITypeStr", "URLSelectionStrategy", "URLSelectionStrategyStr", "ZMQProxyType", "ZMQProxyTypeStr"] +__all__ = ["APIRouterType", "APIRouterTypeStr", "AccumulatorType", "AccumulatorTypeStr", "AccuracyBenchmarkType", "AccuracyBenchmarkTypeStr", "AccuracyGraderType", "AccuracyGraderTypeStr", "AnalyzerType", "AnalyzerTypeStr", "ArrivalPattern", "ArrivalPatternStr", "CommClientType", "CommClientTypeStr", "CommunicationBackend", "CommunicationBackendStr", "ComposerType", "ComposerTypeStr", "ConsoleExporterType", "ConsoleExporterTypeStr", "ConvergenceCriterionType", "ConvergenceCriterionTypeStr", "CustomDatasetType", "CustomDatasetTypeStr", "DataExporterType", "DataExporterTypeStr", "DatasetBackingStoreType", "DatasetBackingStoreTypeStr", "DatasetClientStoreType", "DatasetClientStoreTypeStr", "DatasetFormat", "DatasetFormatStr", "DatasetSamplingStrategy", "DatasetSamplingStrategyStr", "EndpointType", "EndpointTypeStr", "GPUTelemetryCollectorType", "GPUTelemetryCollectorTypeStr", "PhaseType", "PhaseTypeStr", "PlotType", "PlotTypeStr", "PluginType", "PluginTypeStr", "PublicDatasetType", "PublicDatasetTypeStr", "RampType", "RampTypeStr", "RecordProcessorType", "RecordProcessorTypeStr", "SearchPlannerType", "SearchPlannerTypeStr", "SearchRecipePostProcessType", "SearchRecipePostProcessTypeStr", "SearchRecipeType", "SearchRecipeTypeStr", "ServiceRunType", "ServiceRunTypeStr", "ServiceType", "ServiceTypeStr", "StreamExporterType", "StreamExporterTypeStr", "TimingMode", "TimingModeStr", "TransportType", "TransportTypeStr", "UIType", "UITypeStr", "URLSelectionStrategy", "URLSelectionStrategyStr", "ZMQProxyType", "ZMQProxyTypeStr"] # Plugin Protocol Categories if TYPE_CHECKING: @@ -85,6 +85,10 @@ StreamExporterType = plugins.create_enum(PluginType.STREAM_EXPORTER, "StreamExporterType", module=__name__) """Dynamic enum for stream exporter. Example: StreamExporterType.GPU_TELEMETRY_JSONL_WRITER, StreamExporterType.OTEL_METRICS_STREAMER, StreamExporterType.SERVER_METRICS_JSONL_WRITER""" +AnalyzerTypeStr: TypeAlias = str +AnalyzerType = plugins.create_enum(PluginType.ANALYZER, "AnalyzerType", module=__name__) +"""Dynamic enum for analyzer. Example: AnalyzerType.ENERGY_EFFICIENCY""" + AccuracyGraderTypeStr: TypeAlias = str AccuracyGraderType = plugins.create_enum(PluginType.ACCURACY_GRADER, "AccuracyGraderType", module=__name__) """Dynamic enum for accuracy grader. Example: AccuracyGraderType.CODE_EXECUTION, AccuracyGraderType.LIGHTEVAL_GSM8K, AccuracyGraderType.MULTIPLE_CHOICE""" diff --git a/src/aiperf/plugin/plugins.py b/src/aiperf/plugin/plugins.py index f108eddbd6..a624bc7f28 100644 --- a/src/aiperf/plugin/plugins.py +++ b/src/aiperf/plugin/plugins.py @@ -928,7 +928,7 @@ def _load_package_metadata( # ruff: noqa: I001 from aiperf.accuracy.protocols import AccuracyBenchmarkProtocol, AccuracyGraderProtocol from aiperf.api.routers.base_router import BaseRouter - from aiperf.common.accumulator_protocols import AccumulatorProtocol, StreamExporterProtocol + from aiperf.common.accumulator_protocols import AccumulatorProtocol, AnalyzerProtocol, StreamExporterProtocol from aiperf.common.protocols import CommunicationClientProtocol, CommunicationProtocol, ServiceProtocol from aiperf.controller.protocols import ServiceManagerProtocol from aiperf.dataset.composer.base import BaseDatasetComposer @@ -939,7 +939,7 @@ def _load_package_metadata( from aiperf.orchestrator.convergence.base import ConvergenceCriterion from aiperf.orchestrator.search_planner.base import SearchPlanner from aiperf.plot.core.plot_type_handlers import PlotTypeHandlerProtocol - from aiperf.plugin.enums import APIRouterType, AccumulatorType, AccuracyBenchmarkType, AccuracyGraderType, ArrivalPattern, CommClientType, CommunicationBackend, ComposerType, ConsoleExporterType, ConvergenceCriterionType, CustomDatasetType, DataExporterType, DatasetBackingStoreType, DatasetClientStoreType, DatasetSamplingStrategy, EndpointType, GPUTelemetryCollectorType, PlotType, PluginType, PluginTypeStr, PublicDatasetType, RampType, RecordProcessorType, SearchPlannerType, SearchRecipePostProcessType, SearchRecipeType, ServiceRunType, ServiceType, StreamExporterType, TimingMode, TransportType, UIType, URLSelectionStrategy, ZMQProxyType + from aiperf.plugin.enums import APIRouterType, AccumulatorType, AccuracyBenchmarkType, AccuracyGraderType, AnalyzerType, ArrivalPattern, CommClientType, CommunicationBackend, ComposerType, ConsoleExporterType, ConvergenceCriterionType, CustomDatasetType, DataExporterType, DatasetBackingStoreType, DatasetClientStoreType, DatasetSamplingStrategy, EndpointType, GPUTelemetryCollectorType, PlotType, PluginType, PluginTypeStr, PublicDatasetType, RampType, RecordProcessorType, SearchPlannerType, SearchRecipePostProcessType, SearchRecipeType, ServiceRunType, ServiceType, StreamExporterType, TimingMode, TransportType, UIType, URLSelectionStrategy, ZMQProxyType from aiperf.post_processors.protocols import RecordProcessorProtocol from aiperf.search_recipes._base import SearchRecipe from aiperf.search_recipes.post_process import PostProcessHandler @@ -1014,6 +1014,10 @@ def get_class(category: Literal[PluginType.STREAM_EXPORTER, "stream_exporter"], @overload def iter_all(category: Literal[PluginType.STREAM_EXPORTER, "stream_exporter"]) -> Iterator[tuple[PluginEntry, type[StreamExporterProtocol]]]: ... @overload + def get_class(category: Literal[PluginType.ANALYZER, "analyzer"], name_or_class_path: AnalyzerType | str) -> type[AnalyzerProtocol]: ... + @overload + def iter_all(category: Literal[PluginType.ANALYZER, "analyzer"]) -> Iterator[tuple[PluginEntry, type[AnalyzerProtocol]]]: ... + @overload def get_class(category: Literal[PluginType.ACCURACY_GRADER, "accuracy_grader"], name_or_class_path: AccuracyGraderType | str) -> type[AccuracyGraderProtocol]: ... @overload def iter_all(category: Literal[PluginType.ACCURACY_GRADER, "accuracy_grader"]) -> Iterator[tuple[PluginEntry, type[AccuracyGraderProtocol]]]: ... diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index 97159cb7b9..21f60c4fda 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -1079,6 +1079,24 @@ stream_exporter: metadata: record_types: [server_metrics] +# ============================================================================= +# Analyzers +# ============================================================================= +# Summarize-time analyzers that join across accumulators via the SummaryContext. +# ============================================================================= +analyzer: + energy_efficiency: + class: aiperf.metrics.energy_efficiency_analyzer:EnergyEfficiencyAnalyzer + description: | + Joins GPU-telemetry energy/power with inference token totals to emit + energy-efficiency metrics (joules per token, performance per watt, energy + delay product, ...). Skipped when GPU telemetry is not collected. + metadata: + # Live GPU accumulator (windowed energy/power query) + metrics summary + # (token/duration totals). These are AccumulatorType names, not record types. + required_accumulators: [gpu_telemetry] + required_summaries: [metric_results] + # ============================================================================= # Artifact Publishers # ============================================================================= diff --git a/src/aiperf/plugin/schema/plugins.schema.json b/src/aiperf/plugin/schema/plugins.schema.json index a243deb897..8652f31eed 100644 --- a/src/aiperf/plugin/schema/plugins.schema.json +++ b/src/aiperf/plugin/schema/plugins.schema.json @@ -130,6 +130,14 @@ "$ref": "#/$defs/StreamExporterPlugin" } }, + "analyzer": { + "title": "Analyzer Plugins", + "type": "object", + "description": "Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously.", + "additionalProperties": { + "$ref": "#/$defs/AnalyzerPlugin" + } + }, "accuracy_grader": { "title": "Accuracy Grader Plugins", "type": "object", @@ -1173,6 +1181,56 @@ "title": "Stream Exporter Plugin", "description": "Stream exporters write each record to an external sink (e.g. JSONL files) as it\narrives. No summarization dependencies. Finalized after all records are processed.\nOne-to-many mapping: multiple exporters loaded simultaneously." }, + "AnalyzerPlugin": { + "type": "object", + "properties": { + "class": { + "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", + "title": "Class", + "type": "string" + }, + "description": { + "default": "", + "description": "Brief explanation of what this plugin type does and when to use it.", + "title": "Description", + "type": "string" + }, + "priority": { + "default": 0, + "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", + "title": "Priority", + "type": "integer" + }, + "metadata": { + "description": "Metadata schema for analyzer plugins.\n\nAnalyzers run at summarize time and join across accumulators. They store no\nrecords; instead they declare their dependencies by KIND:\n\n- ``required_accumulators``: needs the LIVE accumulator instance (via\n ``SummaryContext.get_accumulator``) to run a query not present in the\n summary \u2014 e.g. energy efficiency calls ``GPUTelemetryAccumulator``'s\n windowed ``total_energy_joules`` / ``total_power_watts``.\n- ``required_summaries``: needs only the already-computed summary output\n (via ``SummaryContext.get_output``) \u2014 e.g. energy efficiency reads token\n and duration totals off the metrics accumulator's summary.\n\nRecordsManager runs an analyzer only when every required accumulator is\nloaded AND every required summary was produced; otherwise it is skipped\n(e.g. energy efficiency is skipped when GPU telemetry is disabled).\n\nReferenced by: categories.yaml analyzer.metadata_class\nUsed in: plugins.yaml analyzer entries", + "properties": { + "required_accumulators": { + "description": "AccumulatorType names whose LIVE instance this analyzer queries via SummaryContext.get_accumulator(). The analyzer is skipped unless all are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics'.", + "items": { + "type": "string" + }, + "title": "Required Accumulators", + "type": "array" + }, + "required_summaries": { + "description": "AccumulatorType names whose SUMMARY output this analyzer reads via SummaryContext.get_output(). The analyzer is skipped unless all were produced. Values: 'metric_records', 'gpu_telemetry', 'server_metrics'.", + "items": { + "type": "string" + }, + "title": "Required Summaries", + "type": "array" + } + }, + "title": "AnalyzerMetadata", + "type": "object" + } + }, + "required": [ + "class" + ], + "title": "Analyzer Plugin", + "description": "Analyzers run at summarize time and join across accumulators, reading peer\nstate from the SummaryContext to compute cross-accumulator metrics (e.g.\nenergy efficiency joins GPU telemetry to inference tokens). They store no\nrecords and declare the accumulators they require; skipped when any is absent.\nOne-to-many mapping: multiple analyzers loaded simultaneously." + }, "AccuracyGraderPlugin": { "type": "object", "properties": { diff --git a/src/aiperf/plugin/schema/schemas.py b/src/aiperf/plugin/schema/schemas.py index 79e7ecce1a..51b5dd1f86 100644 --- a/src/aiperf/plugin/schema/schemas.py +++ b/src/aiperf/plugin/schema/schemas.py @@ -512,6 +512,47 @@ class RecordRoutingMetadata(BaseModel): ) +class AnalyzerMetadata(BaseModel): + """Metadata schema for analyzer plugins. + + Analyzers run at summarize time and join across accumulators. They store no + records; instead they declare their dependencies by KIND: + + - ``required_accumulators``: needs the LIVE accumulator instance (via + ``SummaryContext.get_accumulator``) to run a query not present in the + summary — e.g. energy efficiency calls ``GPUTelemetryAccumulator``'s + windowed ``total_energy_joules`` / ``total_power_watts``. + - ``required_summaries``: needs only the already-computed summary output + (via ``SummaryContext.get_output``) — e.g. energy efficiency reads token + and duration totals off the metrics accumulator's summary. + + RecordsManager runs an analyzer only when every required accumulator is + loaded AND every required summary was produced; otherwise it is skipped + (e.g. energy efficiency is skipped when GPU telemetry is disabled). + + Referenced by: categories.yaml analyzer.metadata_class + Used in: plugins.yaml analyzer entries + """ + + required_accumulators: list[str] = Field( + default_factory=list, + description=( + "AccumulatorType names whose LIVE instance this analyzer queries via " + "SummaryContext.get_accumulator(). The analyzer is skipped unless all " + "are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics'." + ), + ) + + required_summaries: list[str] = Field( + default_factory=list, + description=( + "AccumulatorType names whose SUMMARY output this analyzer reads via " + "SummaryContext.get_output(). The analyzer is skipped unless all were " + "produced. Values: 'metric_records', 'gpu_telemetry', 'server_metrics'." + ), + ) + + # ============================================================================= # Re-exports # ============================================================================= diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 9dfbda34bd..419b59dd49 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -12,6 +12,7 @@ AccumulatorProtocol, ExportContext, StreamExporterProtocol, + SummaryContext, ) from aiperf.common.base_component_service import BaseComponentService from aiperf.common.constants import NANOS_PER_SECOND @@ -57,7 +58,6 @@ TimesliceResult, WorkerProcessingStats, ) -from aiperf.common.models.server_metrics_models import TimeRangeFilter from aiperf.common.utils import yield_to_event_loop from aiperf.config.comm import ZMQDualBindConfig from aiperf.credit.messages import ( @@ -85,8 +85,10 @@ from aiperf.records.dataset_gate import await_dataset_configured from aiperf.records.error_tracker import ErrorTracker from aiperf.records.records_manager_processing import ( + LoadedAnalyzer, generate_realtime_metrics, load_accumulators, + load_analyzers, load_stream_exporters, ) from aiperf.records.records_tracker import RecordsTracker @@ -439,6 +441,9 @@ def __init__( self._stream_exporters: dict[StreamExporterType, StreamExporterProtocol] = ( load_stream_exporters(self) ) + # Summarize-time cross-accumulator analyzers (e.g. energy efficiency), + # each carrying its live-instance and summary dependencies. + self._analyzers: list[LoadedAnalyzer] = load_analyzers(self) self._routing_table = self._build_routing_table() self._log_routing_table() @@ -1034,38 +1039,42 @@ def _maybe_hint_missing_cache_reporting( self._warned_missing_cache_reporting = True self.warning(CACHE_REPORTING_HINT) - def _apply_gpu_efficiency_metrics( - self, - records_results: list[MetricResult], - phase_stats: PhaseRecordsStats, - phase: CreditPhase, - ) -> None: - """Append GPU efficiency metrics to records_results when the phase has a real window. + async def _run_analyzers(self, ctx: SummaryContext) -> list[MetricResult]: + """Run summarize-time analyzer plugins that join across accumulators. - No-op if no accumulator is configured. Skips with a warning when the - phase has no records (either bound is None) — a two-call time.time_ns() - fallback would otherwise yield a zero-width window and power (gauge) - would either emit a misleading 0.0W or be silently dropped depending - on sample jitter. + An analyzer is skipped unless every accumulator it needs a LIVE instance + of (``required_accumulators``) is loaded AND every accumulator whose + SUMMARY it reads (``required_summaries``) was produced — e.g. the + energy-efficiency analyzer queries the live GPU accumulator and reads the + metrics summary. One analyzer's failure is logged and does not abort the + rest. Returns the flattened MetricResults to merge into the summary. """ - if self._gpu_telemetry_accumulator is None: - return - if phase_stats.start_ns is None or phase_stats.requests_end_ns is None: - self.warning( - f"Skipping efficiency metrics for phase {phase}: " - f"start_ns={phase_stats.start_ns}, " - f"requests_end_ns={phase_stats.requests_end_ns}" - ) - return - time_filter = TimeRangeFilter( - start_ns=phase_stats.start_ns, - end_ns=phase_stats.requests_end_ns, - ) - efficiency_metrics = self._gpu_telemetry_accumulator.compute_efficiency_metrics( - metric_results=records_results, - time_filter=time_filter, - ) - records_results.extend(efficiency_metrics) + if not self._analyzers: + return [] + loaded = {str(acc_type) for acc_type in self._accumulators} + summarized = {str(acc_type) for acc_type in ctx.accumulator_outputs} + results: list[MetricResult] = [] + for loaded_analyzer in self._analyzers: + analyzer = loaded_analyzer.analyzer + name = analyzer.__class__.__name__ + missing_acc = [ + r for r in loaded_analyzer.required_accumulators if r not in loaded + ] + missing_sum = [ + r for r in loaded_analyzer.required_summaries if r not in summarized + ] + if missing_acc or missing_sum: + self.debug( + lambda n=name, a=missing_acc, s=missing_sum: ( + f"Skipping analyzer {n}: missing accumulators {a}, summaries {s}" + ) + ) + continue + try: + results.extend(await analyzer.analyze(ctx)) + except Exception as e: # noqa: BLE001 - one analyzer must not abort the summary + self.error(f"Analyzer {name} failed: {e!r}") + return results async def _summarize_one_accumulator( self, @@ -1139,17 +1148,33 @@ def _bucket_accumulator_summary( async def _summarize_metric_record_accumulators( self, phase: CreditPhase, cancelled: bool - ) -> tuple[list[MetricResult], list[TimesliceResult], list[ErrorDetails]]: + ) -> tuple[ + list[MetricResult], list[TimesliceResult], list[ErrorDetails], SummaryContext + ]: """Summarize the metric_records accumulators (the byte-exact engine). Telemetry / server-metrics accumulators are summarized separately via the dedicated ``_publish_telemetry_results`` / ``_publish_server_metrics_results`` side-channels so they are not double-processed here. + + Also returns a populated ``SummaryContext`` — every loaded accumulator + instance plus the metric_records summaries keyed by ``AccumulatorType`` — + so summarize-time ``analyzer`` plugins can join across accumulators + (e.g. energy efficiency joins GPU telemetry to inference tokens). """ records_results: list[MetricResult] = [] timeslices: list[TimesliceResult] = [] error_results: list[ErrorDetails] = [] + phase_stats = self._records_tracker.create_stats_for_phase(phase) + summary_ctx = SummaryContext( + accumulators=dict(self._accumulators), + start_ns=phase_stats.start_ns or 0, + end_ns=phase_stats.requests_end_ns or 0, + phase=phase, + cancelled=cancelled, + ) + # Only the metric_records-typed accumulators feed the summary records. acc_items = [ (acc_type, acc) @@ -1157,9 +1182,8 @@ async def _summarize_metric_record_accumulators( if acc in self._metric_record_accumulators ] if not acc_items: - return records_results, timeslices, error_results + return records_results, timeslices, error_results, summary_ctx - phase_stats = self._records_tracker.create_stats_for_phase(phase) ctx = ExportContext( start_ns=phase_stats.start_ns, end_ns=phase_stats.requests_end_ns, @@ -1175,12 +1199,14 @@ async def _summarize_metric_record_accumulators( return_exceptions=False, ) for acc_type, summary in summaries: + # Expose each accumulator's summary for cross-accumulator analyzers. + summary_ctx.accumulator_outputs[acc_type] = summary ts = self._bucket_accumulator_summary( acc_type, summary, records_results, error_results ) if ts: timeslices = ts - return records_results, timeslices, error_results + return records_results, timeslices, error_results, summary_ctx def _has_records_for_phase(self, phase: CreditPhase) -> bool: phase_trackers = getattr(self._records_tracker, "_phase_trackers", {}) @@ -1200,6 +1226,7 @@ async def _summarize_warmup_metric_records(self) -> list[MetricResult] | None: records_results, _, error_results, + _summary_ctx, ) = await self._summarize_metric_record_accumulators( CreditPhase.WARMUP, self._records_tracker.was_phase_cancelled(CreditPhase.WARMUP), @@ -1318,6 +1345,7 @@ async def _process_results( records_results, timeslices, error_results, + summary_ctx, ) = await self._summarize_metric_record_accumulators(phase, cancelled) warmup_records_results = await self._summarize_warmup_metric_records() @@ -1325,11 +1353,13 @@ async def _process_results( await self._finalize_stream_exporters() phase_stats = self._records_tracker.create_stats_for_phase(phase) - # Snapshot count BEFORE extending with efficiency metrics — `completed` - # reports the number of request-derived records, not derived aggregates. + # Snapshot count BEFORE extending with derived aggregates (efficiency, + # analyzers) — `completed` reports request-derived records only. records_completed = len(records_results) - self._apply_gpu_efficiency_metrics(records_results, phase_stats, phase) + # Cross-accumulator analyzer plugins (e.g. energy efficiency) run after + # all accumulators have summarized, reading peers via the SummaryContext. + records_results.extend(await self._run_analyzers(summary_ctx)) result = ProcessRecordsResult( results=ProfileResults( @@ -1387,7 +1417,7 @@ async def _process_results( self.debug("_process_results completed, returning result") return result - def _process_telemetry_results(self) -> ProcessTelemetryResult: + async def _process_telemetry_results(self) -> ProcessTelemetryResult: """Process telemetry results by exporting the accumulated telemetry data. Returns: @@ -1409,15 +1439,14 @@ def _process_telemetry_results(self) -> ProcessTelemetryResult: ) # Get timing from profiling phase stats - # Note: end_ns is not passed to include the final telemetry scrape that + # Note: end_ns is left None to include the final telemetry scrape that # occurs after PROFILE_COMPLETE but before export_results is called. # If start_ns is None (no profiling phase), include all data. phase_stats = self._records_tracker.create_stats_for_phase( CreditPhase.PROFILING ) - telemetry_export_data = self._gpu_telemetry_accumulator.export_results( - start_ns=phase_stats.start_ns, - error_summary=error_summary, + telemetry_export_data = await self._gpu_telemetry_accumulator.export_results( + ExportContext(start_ns=phase_stats.start_ns, error_summary=error_summary) ) return ProcessTelemetryResult( @@ -1431,7 +1460,7 @@ async def _publish_telemetry_results(self, phase: CreditPhase) -> None: Called at the end of _process_results to keep telemetry separate from inference metrics in the results pipeline. """ - telemetry_result = self._process_telemetry_results() + telemetry_result = await self._process_telemetry_results() await self.publish( ProcessTelemetryResultMessage( service_id=self.service_id, @@ -1471,11 +1500,13 @@ async def _process_server_metrics_results(self) -> ProcessServerMetricsResult: server_metrics_export_data = ( await self._server_metrics_accumulator.export_results( - start_ns=profiling_start_ns, - end_ns=profiling_end_ns, - error_summary=error_summary, - warmup_start_ns=warmup_phase_stats.start_ns, - warmup_end_ns=warmup_phase_stats.requests_end_ns, + ExportContext( + start_ns=profiling_start_ns, + end_ns=profiling_end_ns, + error_summary=error_summary, + warmup_start_ns=warmup_phase_stats.start_ns, + warmup_end_ns=warmup_phase_stats.requests_end_ns, + ) ) ) diff --git a/src/aiperf/records/records_manager_processing.py b/src/aiperf/records/records_manager_processing.py index 4474ee77f0..f1ebe09171 100644 --- a/src/aiperf/records/records_manager_processing.py +++ b/src/aiperf/records/records_manager_processing.py @@ -13,6 +13,7 @@ import asyncio import time +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Protocol from aiperf.common.accumulator_protocols import SummaryContext @@ -36,6 +37,7 @@ if TYPE_CHECKING: from aiperf.common.accumulator_protocols import ( AccumulatorProtocol, + AnalyzerProtocol, StreamExporterProtocol, ) from aiperf.common.models.branch_stats import BranchStats @@ -47,6 +49,21 @@ _logger = AIPerfLogger(__name__) +@dataclass(slots=True) +class LoadedAnalyzer: + """An analyzer plugin instance plus its declared dependencies by kind. + + ``required_accumulators`` names accumulators whose live instance the analyzer + queries (``SummaryContext.get_accumulator``); ``required_summaries`` names + accumulators whose summary output it reads (``SummaryContext.get_output``). + RecordsManager runs the analyzer only when both are satisfied. + """ + + analyzer: AnalyzerProtocol + required_accumulators: list[str] = field(default_factory=list) + required_summaries: list[str] = field(default_factory=list) + + class _LoaderHost(Protocol): """Minimal surface the plugin loaders use on the owning service.""" @@ -133,6 +150,60 @@ def load_stream_exporters( return exporters +def load_analyzers(host: _LoaderHost) -> list[LoadedAnalyzer]: + """Instantiate all enabled ``ANALYZER`` plugins for ``host``. + + Analyzers are stateless summarize-time components (no lifecycle, no record + ingestion) that read peer accumulators via the SummaryContext. Each entry is + returned as a :class:`LoadedAnalyzer` carrying its declared dependencies by + kind — ``required_accumulators`` (live instance) and ``required_summaries`` + (summary output) — so the caller can skip an analyzer whose dependencies are + unavailable. Same disable/error policy as :func:`load_accumulators`. + """ + analyzers: list[LoadedAnalyzer] = [] + for entry in plugins.iter_entries(PluginType.ANALYZER): + try: + AnalyzerClass = plugins.get_class(PluginType.ANALYZER, entry.name) + analyzer = AnalyzerClass( + service_id=host.service_id, + run=host.run, + pub_client=host.pub_client, + ) + loaded = LoadedAnalyzer( + analyzer=analyzer, + required_accumulators=list( + entry.metadata.get("required_accumulators", []) + ), + required_summaries=list(entry.metadata.get("required_summaries", [])), + ) + # Catch metadata typos loudly: a required name that is not a known + # AccumulatorType would otherwise silently disable the analyzer at + # every run (its dependency never "resolves"). + known = {str(t) for t in AccumulatorType} + unknown = [ + r + for r in (*loaded.required_accumulators, *loaded.required_summaries) + if r not in known + ] + if unknown: + host.error( + f"Analyzer {entry.name} declares unknown accumulator dependencies " + f"{unknown} (valid: {sorted(known)}); it will never run. Fix the " + "required_accumulators/required_summaries in plugins.yaml." + ) + analyzers.append(loaded) + host.debug( + f"Created analyzer: {entry.name}: {analyzer.__class__.__name__} " + f"(accumulators={loaded.required_accumulators}, " + f"summaries={loaded.required_summaries})" + ) + except (PluginDisabled, PostProcessorDisabled): + host.debug(f"Analyzer {entry.name} is disabled and will not be used") + except Exception as e: # noqa: BLE001 - one bad analyzer must not abort the records manager + host.error(f"Failed to create analyzer {entry.name}: {e}") + return analyzers + + async def generate_realtime_metrics( accumulators: list[AccumulatorProtocol], timeout: float = 30.0, diff --git a/src/aiperf/server_metrics/accumulator.py b/src/aiperf/server_metrics/accumulator.py index fe5abac185..7a5abebcf6 100644 --- a/src/aiperf/server_metrics/accumulator.py +++ b/src/aiperf/server_metrics/accumulator.py @@ -17,7 +17,6 @@ from aiperf.common.exceptions import DataExporterDisabled, PostProcessorDisabled from aiperf.common.growable_array import GrowableArray from aiperf.common.models import MetricResult -from aiperf.common.models.error_models import ErrorDetailsCount from aiperf.common.models.server_metrics_models import ( CounterMetricData, GaugeMetricData, @@ -39,6 +38,7 @@ ) if TYPE_CHECKING: + from aiperf.common.accumulator_protocols import ExportContext, SummaryContext from aiperf.config.resolution.plan import BenchmarkRun _METRIC_DATA_CLASSES: dict[ @@ -131,34 +131,27 @@ def query_time_range(self, start_ns: int, end_ns: int) -> NDArray[np.bool_]: ts = self._timestamps_ns.data return (ts >= start_ns) & (ts < end_ns) - async def export_results( - self, - start_ns: int, - end_ns: int, - error_summary: list[ErrorDetailsCount] | None = None, - *, - warmup_start_ns: int | None = None, - warmup_end_ns: int | None = None, - ) -> ServerMetricsResults | None: + async def export_results(self, ctx: ExportContext) -> ServerMetricsResults | None: """Export accumulated server metrics as results for final reporting. Called at the end of profiling to generate the final ServerMetricsResults object containing all computed statistics. Applies time filtering to exclude warmup periods and computes per-endpoint summaries with stats. - The time range [start_ns, end_ns] represents the profiling phase only, - excluding warmup. Reference points before start_ns are used for counter - and histogram delta calculations. - - Args: - start_ns: Profiling phase start time in nanoseconds (excludes warmup period) - end_ns: Profiling phase end time in nanoseconds (may extend beyond last collection) - error_summary: Optional list of error counts from collection failures + Reads the profiling window from ``ctx.start_ns/ctx.end_ns`` (excludes + warmup; reference points before start_ns drive counter/histogram deltas) + and the warmup window from ``ctx.warmup_start_ns/ctx.warmup_end_ns``. Returns: ServerMetricsResults containing endpoint summaries with computed statistics, or None if no endpoints were successfully scraped during profiling. """ + start_ns = ctx.start_ns + end_ns = ctx.end_ns + error_summary = ctx.error_summary + warmup_start_ns = ctx.warmup_start_ns + warmup_end_ns = ctx.warmup_end_ns + if not self._server_metrics_hierarchy.endpoints: return None @@ -393,7 +386,7 @@ async def _export_parquet_if_enabled(self, time_filter: TimeRangeFilter) -> None except Exception as e: self.error(f"Failed to export server metrics to Parquet: {e!r}") - async def summarize(self) -> list[MetricResult]: + async def summarize(self, ctx: SummaryContext | None = None) -> list[MetricResult]: """Summarize accumulated metrics into MetricResult list. Server metrics are exported separately via export_results() rather than diff --git a/tests/unit/gpu_telemetry/test_accumulator.py b/tests/unit/gpu_telemetry/test_accumulator.py index 02fd080c38..e9e5abdb3f 100644 --- a/tests/unit/gpu_telemetry/test_accumulator.py +++ b/tests/unit/gpu_telemetry/test_accumulator.py @@ -5,11 +5,8 @@ import pytest -from aiperf.common.enums import EnergyMetricUnit, GenericMetricUnit, PowerMetricUnit -from aiperf.common.environment import Environment from aiperf.common.exceptions import NoMetricValue from aiperf.common.models import MetricResult -from aiperf.common.models.server_metrics_models import TimeRangeFilter from aiperf.common.models.telemetry_models import ( GpuMetadata, GpuTelemetryData, @@ -384,392 +381,3 @@ async def test_summarize_multiple_gpus( assert len(gpu0_results) > 0 assert len(gpu1_results) > 0 - - -class TestComputeEfficiencyMetrics: - """Test GPUTelemetryAccumulator.compute_efficiency_metrics.""" - - @pytest.fixture - def accumulator( - self, - mock_run, - mock_pub_client, - ) -> GPUTelemetryAccumulator: - return GPUTelemetryAccumulator( - run=mock_run, - pub_client=mock_pub_client, - ) - - @pytest.fixture - def time_filter(self) -> TimeRangeFilter: - return TimeRangeFilter(start_ns=2_000_000_000, end_ns=5_000_000_000) - - def _make_gpu_mock( - self, - power_avg: float | None = None, - energy_delta_mj: float | None = None, - ) -> Mock: - """Return a GpuTelemetryData mock with controllable metric results.""" - metadata = GpuMetadata( - gpu_index=0, - gpu_uuid="GPU-test-uuid-0000", - gpu_model_name="Test GPU", - ) - gpu = Mock(spec=GpuTelemetryData) - gpu.metadata = metadata - - def get_metric_result( - metric_name: str, - tag: str, - header: str, - unit: str, - time_filter: TimeRangeFilter | None = None, - is_counter: bool = False, - ) -> MetricResult: - if metric_name == "gpu_power_usage": - if power_avg is None: - raise NoMetricValue("No power data") - return MetricResult( - tag=tag, header=header, unit=unit, avg=power_avg, count=3 - ) - if metric_name == "energy_consumption": - if energy_delta_mj is None: - raise NoMetricValue("No energy data") - return MetricResult( - tag=tag, header=header, unit=unit, avg=energy_delta_mj - ) - raise NoMetricValue(f"No data for {metric_name}") - - gpu.get_metric_result.side_effect = get_metric_result - return gpu - - def test_happy_path_all_metrics_present( - self, accumulator: GPUTelemetryAccumulator, time_filter: TimeRangeFilter - ) -> None: - """power + energy + tokens + concurrency=1 (default) → four MetricResults.""" - gpu = self._make_gpu_mock( - power_avg=200.0, energy_delta_mj=0.001 - ) # 0.001 MJ = 1000 J - accumulator._hierarchy.dcgm_endpoints = { - "http://node1:9401/metrics": {"GPU-test": gpu} - } - metric_results = [ - MetricResult( - tag="total_output_tokens", - header="Total Output Tokens", - unit="tokens", - avg=2000.0, - ) - ] - - results = accumulator.compute_efficiency_metrics(metric_results, time_filter) - - tags = {r.tag for r in results} - assert tags == { - "total_gpu_power", - "total_gpu_energy", - "output_tokens_per_joule", - "energy_per_user", - } - - power = next(r for r in results if r.tag == "total_gpu_power") - assert power.avg == pytest.approx(200.0) - assert power.unit == str(PowerMetricUnit.WATT) - assert power.header == "Total GPU Power (1 GPU)" - - energy = next(r for r in results if r.tag == "total_gpu_energy") - assert energy.avg == pytest.approx(1000.0) # 0.001 MJ → J - assert energy.unit == str(EnergyMetricUnit.JOULE) - assert energy.header == "Total GPU Energy (1 GPU)" - - tpj = next(r for r in results if r.tag == "output_tokens_per_joule") - assert tpj.avg == pytest.approx(2.0) # 2000 tokens / 1000 J - assert tpj.unit == str(GenericMetricUnit.TOKENS_PER_JOULE) - assert tpj.header == "Output Tokens per Joule (1 GPU)" - - epu = next(r for r in results if r.tag == "energy_per_user") - assert epu.avg == pytest.approx(1000.0) # 1000 J / 1 user (default) - assert epu.unit == str(GenericMetricUnit.JOULES_PER_USER) - assert epu.header == "Energy per User (1 GPU)" - - def test_energy_per_user_scales_with_concurrency( - self, accumulator: GPUTelemetryAccumulator, time_filter: TimeRangeFilter - ) -> None: - """concurrency=N (positive int) → emit total_energy / N.""" - accumulator.run.cfg.get_profiling_phases()[0].concurrency = 8 - gpu = self._make_gpu_mock(power_avg=200.0, energy_delta_mj=0.001) # 1000 J - accumulator._hierarchy.dcgm_endpoints = { - "http://node1:9401/metrics": {"GPU-test": gpu} - } - - results = accumulator.compute_efficiency_metrics([], time_filter) - - epu = next(r for r in results if r.tag == "energy_per_user") - assert epu.avg == pytest.approx(125.0) # 1000 J / 8 users - assert epu.unit == str(GenericMetricUnit.JOULES_PER_USER) - assert epu.header == "Energy per User (1 GPU)" - - def test_energy_per_user_omitted_when_concurrency_none( - self, accumulator: GPUTelemetryAccumulator, time_filter: TimeRangeFilter - ) -> None: - """concurrency=None (e.g. pure request-rate run) → energy_per_user omitted.""" - accumulator.run.cfg.get_profiling_phases()[0].concurrency = None - gpu = self._make_gpu_mock(power_avg=200.0, energy_delta_mj=0.001) - accumulator._hierarchy.dcgm_endpoints = { - "http://node1:9401/metrics": {"GPU-test": gpu} - } - - results = accumulator.compute_efficiency_metrics([], time_filter) - - tags = {r.tag for r in results} - assert "energy_per_user" not in tags - assert "total_gpu_energy" in tags # sibling still emits - - def test_energy_per_user_omitted_when_no_energy_data( - self, accumulator: GPUTelemetryAccumulator, time_filter: TimeRangeFilter - ) -> None: - """concurrency set but no GPU energy → energy_per_user omitted (no numerator).""" - accumulator.run.cfg.get_profiling_phases()[0].concurrency = 8 - gpu = self._make_gpu_mock(power_avg=150.0, energy_delta_mj=None) - accumulator._hierarchy.dcgm_endpoints = { - "http://node1:9401/metrics": {"GPU-test": gpu} - } - - results = accumulator.compute_efficiency_metrics([], time_filter) - - tags = {r.tag for r in results} - assert "energy_per_user" not in tags - assert "total_gpu_energy" not in tags - - def test_emitted_units_match_metric_class_units( - self, accumulator: GPUTelemetryAccumulator, time_filter: TimeRangeFilter - ) -> None: - """Accumulator-emitted unit strings must equal str(MetricClass.unit) per tag. - - Locks the relationship rather than the literal — a rename of any of - the unit enums would break this without needing per-test updates. - """ - from aiperf.metrics.types.power_efficiency_metrics import ( - EnergyPerUserMetric, - OutputTokensPerJouleMetric, - TotalGpuEnergyMetric, - TotalGpuPowerMetric, - ) - - gpu = self._make_gpu_mock(power_avg=200.0, energy_delta_mj=0.001) - accumulator._hierarchy.dcgm_endpoints = { - "http://node1:9401/metrics": {"GPU-test": gpu} - } - metric_results = [ - MetricResult( - tag="total_output_tokens", header="h", unit="tokens", avg=2000.0 - ) - ] - - results = accumulator.compute_efficiency_metrics(metric_results, time_filter) - by_tag = {r.tag: r for r in results} - - expected = { - TotalGpuPowerMetric.tag: str(TotalGpuPowerMetric.unit), - TotalGpuEnergyMetric.tag: str(TotalGpuEnergyMetric.unit), - OutputTokensPerJouleMetric.tag: str(OutputTokensPerJouleMetric.unit), - EnergyPerUserMetric.tag: str(EnergyPerUserMetric.unit), - } - for tag, expected_unit in expected.items(): - assert by_tag[tag].unit == expected_unit, ( - f"unit drift for {tag}: emitted={by_tag[tag].unit!r}, " - f"metric class={expected_unit!r}" - ) - - def test_no_energy_data_omits_energy_and_tokens_per_joule( - self, accumulator: GPUTelemetryAccumulator, time_filter: TimeRangeFilter - ) -> None: - """No energy data → only power metric returned; tokens/J absent.""" - gpu = self._make_gpu_mock(power_avg=150.0, energy_delta_mj=None) - accumulator._hierarchy.dcgm_endpoints = { - "http://node1:9401/metrics": {"GPU-test": gpu} - } - metric_results = [ - MetricResult(tag="total_output_tokens", header="h", unit="t", avg=1000.0) - ] - - results = accumulator.compute_efficiency_metrics(metric_results, time_filter) - - tags = {r.tag for r in results} - assert "total_gpu_power" in tags - assert "total_gpu_energy" not in tags - assert "output_tokens_per_joule" not in tags - - def test_no_gpu_data_returns_empty_list( - self, accumulator: GPUTelemetryAccumulator, time_filter: TimeRangeFilter - ) -> None: - """No GPU data → empty list returned without error.""" - results = accumulator.compute_efficiency_metrics([], time_filter) - assert results == [] - - def test_missing_total_output_tokens_omits_tokens_per_joule( - self, accumulator: GPUTelemetryAccumulator, time_filter: TimeRangeFilter - ) -> None: - """total_output_tokens absent from metric_results → tokens/J absent, no error.""" - gpu = self._make_gpu_mock(power_avg=200.0, energy_delta_mj=0.001) - accumulator._hierarchy.dcgm_endpoints = { - "http://node1:9401/metrics": {"GPU-test": gpu} - } - - results = accumulator.compute_efficiency_metrics([], time_filter) - - tags = {r.tag for r in results} - assert "total_gpu_power" in tags - assert "total_gpu_energy" in tags - assert "output_tokens_per_joule" not in tags - - def test_multiple_gpus_sums_power_and_energy( - self, accumulator: GPUTelemetryAccumulator, time_filter: TimeRangeFilter - ) -> None: - """Multiple GPUs across endpoints → power and energy summed.""" - gpu0 = self._make_gpu_mock(power_avg=100.0, energy_delta_mj=0.0005) # 500 J - gpu1 = self._make_gpu_mock(power_avg=150.0, energy_delta_mj=0.0005) # 500 J - accumulator._hierarchy.dcgm_endpoints = { - "http://node1:9401/metrics": {"GPU-0": gpu0, "GPU-1": gpu1} - } - metric_results = [ - MetricResult(tag="total_output_tokens", header="h", unit="t", avg=1000.0) - ] - - results = accumulator.compute_efficiency_metrics(metric_results, time_filter) - - power = next(r for r in results if r.tag == "total_gpu_power") - assert power.avg == pytest.approx(250.0) # 100 + 150 - assert power.count is None - assert power.header == "Total GPU Power (2 GPUs)" - - energy = next(r for r in results if r.tag == "total_gpu_energy") - assert energy.avg == pytest.approx(1000.0) # 500 + 500 - assert energy.count is None - assert energy.header == "Total GPU Energy (2 GPUs)" - - tpj = next(r for r in results if r.tag == "output_tokens_per_joule") - assert tpj.avg == pytest.approx(1.0) # 1000 tokens / 1000 J - assert tpj.header == "Output Tokens per Joule (2 GPUs)" - - epu = next(r for r in results if r.tag == "energy_per_user") - assert epu.avg == pytest.approx(1000.0) # 1000 J / 1 user (default) - assert epu.header == "Energy per User (2 GPUs)" - - def test_header_reflects_partial_cohort_count( - self, accumulator: GPUTelemetryAccumulator, time_filter: TimeRangeFilter - ) -> None: - """Header surfaces the *valid-data* GPU count, not the cohort total. - - Two GPUs are configured but only one reports energy; the total_gpu_energy - header must read "(1 GPU)" so a "1 of 2" partial run is distinguishable - from a "2 of 2" full run. MetricResult.count cannot carry this because - to_json_result strips count to None for DERIVED metrics. - """ - gpu_full = self._make_gpu_mock(power_avg=100.0, energy_delta_mj=0.001) - gpu_power_only = self._make_gpu_mock(power_avg=150.0, energy_delta_mj=None) - accumulator._hierarchy.dcgm_endpoints = { - "http://node1:9401/metrics": {"GPU-0": gpu_full, "GPU-1": gpu_power_only} - } - metric_results = [ - MetricResult(tag="total_output_tokens", header="h", unit="t", avg=2000.0) - ] - - results = accumulator.compute_efficiency_metrics(metric_results, time_filter) - by_tag = {r.tag: r for r in results} - - assert by_tag["total_gpu_power"].header == "Total GPU Power (2 GPUs)" - assert by_tag["total_gpu_energy"].header == "Total GPU Energy (1 GPU)" - assert by_tag["output_tokens_per_joule"].header == ( - "Output Tokens per Joule (1 GPU)" - ) - # energy_per_user inherits the energy-side count (its denominator). - assert by_tag["energy_per_user"].header == "Energy per User (1 GPU)" - - def test_energy_filter_widens_end_ns_by_grace_while_power_filter_stays_bounded( - self, accumulator: GPUTelemetryAccumulator, time_filter: TimeRangeFilter - ) -> None: - # Counter-based energy widens end_ns by FINAL_SCRAPE_GRACE_NS so the - # trailing scrape (which lands after requests_end_ns on the - # COLLECTION_INTERVAL cadence) is captured, but the window stays - # bounded so cooldown, idle, or subsequent-phase samples don't leak - # into the delta. Gauge-based power stays at the unwidened end_ns so - # post-bench idle samples don't drag the average down. - gpu = self._make_gpu_mock(power_avg=200.0, energy_delta_mj=0.001) - accumulator._hierarchy.dcgm_endpoints = { - "http://node1:9401/metrics": {"GPU-test": gpu} - } - metric_results = [ - MetricResult( - tag="total_output_tokens", - header="Total Output Tokens", - unit="tokens", - avg=2000.0, - ) - ] - - accumulator.compute_efficiency_metrics(metric_results, time_filter) - - filters_by_metric: dict[str, TimeRangeFilter] = { - call.args[0]: call.kwargs["time_filter"] - for call in gpu.get_metric_result.call_args_list - } - - power_filter = filters_by_metric["gpu_power_usage"] - assert power_filter.start_ns == time_filter.start_ns - assert power_filter.end_ns == time_filter.end_ns - - energy_filter = filters_by_metric["energy_consumption"] - assert energy_filter.start_ns == time_filter.start_ns - assert energy_filter.end_ns == ( - time_filter.end_ns + Environment.GPU.FINAL_SCRAPE_GRACE_NS - ) - assert energy_filter.end_ns is not None, ( - "energy filter must remain bounded so a multi-phase run cannot leak " - "cooldown or subsequent-phase samples into phase N's energy delta" - ) - - def test_repeated_calls_use_bounded_energy_window_per_phase( - self, accumulator: GPUTelemetryAccumulator - ) -> None: - """Each `compute_efficiency_metrics` call must bound the energy filter at - `phase.end_ns + grace` — never reach into a later phase's samples. - - Multi-phase regression guard: simulate WARMUP -> PROFILING by calling - the method twice with non-overlapping windows. Both calls must produce - bounded energy filters whose `end_ns` reflects the phase being closed, - not "now" or the union of all stored samples. - """ - gpu = self._make_gpu_mock(power_avg=100.0, energy_delta_mj=0.0005) - accumulator._hierarchy.dcgm_endpoints = { - "http://node1:9401/metrics": {"GPU-test": gpu} - } - metric_results = [ - MetricResult(tag="total_output_tokens", header="h", unit="t", avg=500.0) - ] - phase1 = TimeRangeFilter(start_ns=1_000_000_000, end_ns=2_000_000_000) - phase2 = TimeRangeFilter(start_ns=3_000_000_000, end_ns=4_000_000_000) - - accumulator.compute_efficiency_metrics(metric_results, phase1) - accumulator.compute_efficiency_metrics(metric_results, phase2) - - energy_filters = [ - call.kwargs["time_filter"] - for call in gpu.get_metric_result.call_args_list - if call.args[0] == "energy_consumption" - ] - assert len(energy_filters) == 2 - - grace = Environment.GPU.FINAL_SCRAPE_GRACE_NS - assert energy_filters[0].start_ns == phase1.start_ns - assert energy_filters[0].end_ns == phase1.end_ns + grace - assert energy_filters[1].start_ns == phase2.start_ns - assert energy_filters[1].end_ns == phase2.end_ns + grace - - # Phase 1's bounded end must not extend into phase 2 — that's the leak - # the bound prevents. Confirms grace is intentionally small. - assert energy_filters[0].end_ns < phase2.start_ns, ( - f"phase 1 energy window ({energy_filters[0].end_ns}) extends past " - f"phase 2 start ({phase2.start_ns}); the grace window is too large " - f"for safe multi-phase use" - ) diff --git a/tests/unit/metrics/test_energy_efficiency_analyzer.py b/tests/unit/metrics/test_energy_efficiency_analyzer.py new file mode 100644 index 0000000000..568449f3d4 --- /dev/null +++ b/tests/unit/metrics/test_energy_efficiency_analyzer.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Formula + dependency-gating tests for the EnergyEfficiencyAnalyzer. + +The analyzer joins a live GPU-telemetry accumulator (windowed energy/power) with +the metrics-accumulator summary (token/throughput/latency totals) via a +SummaryContext, and emits the energy metric family. Values are pinned to the +formulas in design doc ``0005-energy-efficiency-metrics.md``. +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from aiperf.common.accumulator_protocols import SummaryContext +from aiperf.common.models import MetricResult +from aiperf.metrics.accumulator_models import AccumulatorMetricsSummary +from aiperf.metrics.energy_efficiency_analyzer import EnergyEfficiencyAnalyzer +from aiperf.plugin.enums import AccumulatorType + + +class _StubGpu: + """Minimal GPUTelemetryAccumulator query surface for the analyzer.""" + + def __init__(self, energy: tuple[float, int], power: tuple[float, int]) -> None: + self._energy = energy + self._power = power + + def total_energy_joules(self, start_ns, end_ns) -> tuple[float, int]: + return self._energy + + def total_power_watts(self, start_ns, end_ns) -> tuple[float, int]: + return self._power + + +def _metrics_summary(**avgs: float) -> AccumulatorMetricsSummary: + return AccumulatorMetricsSummary( + results={ + tag: MetricResult(tag=tag, header=tag, unit="x", avg=avg) + for tag, avg in avgs.items() + } + ) + + +def _analyzer(concurrency: int | None = 8) -> EnergyEfficiencyAnalyzer: + run = MagicMock() + run.cfg.get_profiling_phases.return_value = [ + SimpleNamespace(concurrency=concurrency) + ] + return EnergyEfficiencyAnalyzer(service_id="t", run=run, pub_client=None) + + +def _ctx(gpu, summary) -> SummaryContext: + return SummaryContext( + accumulators={AccumulatorType.GPU_TELEMETRY: gpu} if gpu else {}, + accumulator_outputs=( + {AccumulatorType.METRIC_RESULTS: summary} if summary else {} + ), + start_ns=1_000, + end_ns=2_000, + ) + + +class TestEnergyEfficiencyAnalyzer: + @pytest.mark.asyncio + async def test_full_metric_set_matches_doc_formulas(self) -> None: + gpu = _StubGpu(energy=(1000.0, 2), power=(200.0, 2)) + summary = _metrics_summary( + total_output_tokens=5000.0, + total_isl=3000.0, + request_count=100.0, + request_throughput=10.0, + request_latency=500.0, # ms + ) + + results = await _analyzer(concurrency=8).analyze(_ctx(gpu, summary)) + by_tag = {r.tag: r.avg for r in results} + + assert by_tag["total_gpu_power"] == pytest.approx(200.0) + assert by_tag["average_gpu_power"] == pytest.approx(100.0) # 200 / 2 GPUs + assert by_tag["total_gpu_energy"] == pytest.approx(1000.0) + assert by_tag["output_tokens_per_joule"] == pytest.approx(5.0) # 5000 / 1000 + assert by_tag["energy_per_output_token"] == pytest.approx( + 200.0 + ) # 1000*1000/5000 + assert by_tag["energy_per_total_token"] == pytest.approx( + 125.0 + ) # 1e6/(3000+5000) + assert by_tag["energy_per_request"] == pytest.approx(10.0) # 1000 / 100 + assert by_tag["energy_per_user"] == pytest.approx(125.0) # 1000 / 8 + assert by_tag["energy_delay_product"] == pytest.approx(5.0) # 10 * 0.5s + assert by_tag["performance_per_watt"] == pytest.approx(0.05) # 10 / 200 + + @pytest.mark.asyncio + async def test_no_gpu_accumulator_returns_empty(self) -> None: + summary = _metrics_summary(total_output_tokens=5000.0) + results = await _analyzer().analyze(_ctx(None, summary)) + assert results == [] + + @pytest.mark.asyncio + async def test_no_metrics_summary_returns_empty(self) -> None: + gpu = _StubGpu(energy=(1000.0, 2), power=(200.0, 2)) + results = await _analyzer().analyze(_ctx(gpu, None)) + assert results == [] + + @pytest.mark.asyncio + async def test_missing_signals_omit_dependent_metrics(self) -> None: + # No energy signal (count 0) and no tokens: only power-derived metrics. + gpu = _StubGpu(energy=(0.0, 0), power=(150.0, 3)) + summary = _metrics_summary(request_throughput=6.0) + results = await _analyzer(concurrency=None).analyze(_ctx(gpu, summary)) + by_tag = {r.tag: r.avg for r in results} + + assert by_tag["total_gpu_power"] == pytest.approx(150.0) + assert by_tag["average_gpu_power"] == pytest.approx(50.0) + assert by_tag["performance_per_watt"] == pytest.approx(0.04) # 6 / 150 + assert "total_gpu_energy" not in by_tag + assert "energy_per_output_token" not in by_tag + assert "energy_per_user" not in by_tag + + @pytest.mark.asyncio + async def test_concurrency_gates_energy_per_user(self) -> None: + gpu = _StubGpu(energy=(1000.0, 1), power=(100.0, 1)) + summary = _metrics_summary(total_output_tokens=1000.0) + results = await _analyzer(concurrency=None).analyze(_ctx(gpu, summary)) + assert "energy_per_user" not in {r.tag for r in results} diff --git a/tests/unit/metrics/test_power_efficiency_metrics.py b/tests/unit/metrics/test_power_efficiency_metrics.py index aab2869faf..f8482d39a7 100644 --- a/tests/unit/metrics/test_power_efficiency_metrics.py +++ b/tests/unit/metrics/test_power_efficiency_metrics.py @@ -6,34 +6,48 @@ from aiperf.common.exceptions import NoMetricValue from aiperf.metrics.metric_dicts import MetricResultsDict from aiperf.metrics.types.power_efficiency_metrics import ( + AverageGpuPowerMetric, + EnergyDelayProductMetric, + EnergyPerOutputTokenMetric, + EnergyPerRequestMetric, + EnergyPerTotalTokenMetric, EnergyPerUserMetric, OutputTokensPerJouleMetric, + PerformancePerWattMetric, TotalGpuEnergyMetric, TotalGpuPowerMetric, ) +_ENERGY_METRIC_CLASSES = [ + AverageGpuPowerMetric, + EnergyDelayProductMetric, + EnergyPerOutputTokenMetric, + EnergyPerRequestMetric, + EnergyPerTotalTokenMetric, + EnergyPerUserMetric, + OutputTokensPerJouleMetric, + PerformancePerWattMetric, + TotalGpuEnergyMetric, + TotalGpuPowerMetric, +] + class TestPowerEfficiencyDeriveValueContract: - """Pin the `_derive_value` invariant for externally-injected derived metrics. - - The three power-efficiency classes inherit `BaseDerivedMetric` for registry - integration but their values are produced by - `GPUTelemetryAccumulator.compute_efficiency_metrics`, not by the metrics - accumulator's derived-metric pass. Calling - `_derive_value` directly must raise `NoMetricValue` with a message that - names the tag, the operation, and the injection site — so a future + """Pin the `_derive_value` invariant for the analyzer-injected energy metrics. + + The energy-efficiency classes inherit `BaseDerivedMetric` for registry + integration, but their values are produced by + `EnergyEfficiencyAnalyzer.analyze`, not by the metrics accumulator's + derived-metric pass (they need GPU telemetry from a different accumulator). + Calling `_derive_value` directly must raise `NoMetricValue` with a message + that names the tag, the operation, and the injection site — so a future contributor copy-pasting this as the "derived metric pattern" sees the contract spelled out rather than a silent miscalculation. """ @pytest.mark.parametrize( "metric_class", - [ - TotalGpuPowerMetric, - TotalGpuEnergyMetric, - OutputTokensPerJouleMetric, - EnergyPerUserMetric, - ], + _ENERGY_METRIC_CLASSES, ids=lambda c: c.tag, ) def test_derive_value_raises_no_metric_value(self, metric_class) -> None: @@ -48,7 +62,7 @@ def test_derive_value_raises_no_metric_value(self, metric_class) -> None: "error message must name the operation source so agents understand " "which derivation path is being rejected" ) - assert "compute_efficiency_metrics" in msg, ( + assert "EnergyEfficiencyAnalyzer" in msg, ( "error message must point to the actual injection site so a future " "contributor doesn't copy this as the derived-metric pattern" ) diff --git a/tests/unit/records/test_records_manager.py b/tests/unit/records/test_records_manager.py index 4b2684a928..4d82859b25 100644 --- a/tests/unit/records/test_records_manager.py +++ b/tests/unit/records/test_records_manager.py @@ -41,6 +41,7 @@ from aiperf.plugin.enums import AccumulatorType, TimingMode from aiperf.records.error_tracker import ErrorTracker from aiperf.records.records_manager import ErrorTrackingState, RecordsManager +from aiperf.records.records_manager_processing import LoadedAnalyzer from aiperf.records.records_tracker import RecordsTracker from aiperf.timing.config import CreditPhaseConfig @@ -604,11 +605,12 @@ async def test_dispatch_errors_still_update_tracker_and_converge_barrier( ) -class TestRecordsManagerEfficiencyMetricsSnapshot: - """Pin the invariant that `completed` counts request-derived records only.""" +class TestRecordsManagerAnalyzerMetrics: + """Pin the invariant that `completed` counts request-derived records only, + and that analyzer-injected metrics are merged after the snapshot.""" @pytest.mark.asyncio - async def test_completed_excludes_efficiency_metrics(self) -> None: + async def test_completed_excludes_analyzer_metrics(self) -> None: manager = RecordsManager.__new__(RecordsManager) manager.debug = MagicMock() @@ -637,19 +639,28 @@ async def test_completed_excludes_efficiency_metrics(self) -> None: manager._accumulators = {AccumulatorType.METRIC_RESULTS: metric_accumulator} manager._metric_record_accumulators = [metric_accumulator] manager._stream_exporters = {} + manager._gpu_telemetry_accumulator = None + manager._server_metrics_accumulator = None - efficiency_metrics = [ + # An analyzer contributes derived aggregates that must NOT inflate + # `completed` (which counts request-derived records only). + analyzer_metrics = [ MetricResult(tag="total_gpu_power", header="h", unit="W", avg=200.0), MetricResult(tag="total_gpu_energy", header="h", unit="J", avg=1000.0), MetricResult( tag="output_tokens_per_joule", header="h", unit="tokens/J", avg=0.002 ), ] - accumulator = MagicMock() - accumulator.compute_efficiency_metrics = MagicMock( - return_value=efficiency_metrics - ) - manager._gpu_telemetry_accumulator = accumulator + stub_analyzer = MagicMock() + stub_analyzer.analyze = AsyncMock(return_value=analyzer_metrics) + manager._analyzers = [ + LoadedAnalyzer( + analyzer=stub_analyzer, + required_accumulators=[], + required_summaries=[], + ) + ] + manager._run_analyzers = RecordsManager._run_analyzers.__get__(manager) manager._records_tracker = MagicMock() manager._records_tracker.create_stats_for_phase.return_value = MagicMock( @@ -665,7 +676,7 @@ async def test_completed_excludes_efficiency_metrics(self) -> None: assert result.results.completed == len(request_records) assert len(result.results.records) == len(request_records) + len( - efficiency_metrics + analyzer_metrics ) assert {r.tag for r in result.results.records} == { "request_latency", @@ -674,115 +685,7 @@ async def test_completed_excludes_efficiency_metrics(self) -> None: "total_gpu_energy", "output_tokens_per_joule", } - - -class TestRecordsManagerEfficiencyMetricsDegeneratePhase: - """Efficiency metrics need a real record-derived phase window.""" - - @pytest.mark.asyncio - async def test_none_phase_window_skips_efficiency_metrics_with_warning( - self, - ) -> None: - manager = RecordsManager.__new__(RecordsManager) - - manager.debug = MagicMock() - manager.info = MagicMock() - manager.warning = MagicMock() - manager.error = MagicMock() - manager.exception = MagicMock() - manager.service_id = "records-manager-test" - manager._latest_branch_stats = None - manager.publish = AsyncMock() - - manager.run = MagicMock() - manager.run.cfg.gpu_telemetry_disabled = True - manager.run.cfg.server_metrics_disabled = True - manager.run.cfg.network_latency.enabled = False - - request_records = [ - MetricResult(tag="request_latency", header="h", unit="ms", avg=1.0), - ] - metric_accumulator = MagicMock() - metric_accumulator.summarize = AsyncMock( - return_value=AccumulatorMetricsSummary( - results={r.tag: r for r in request_records}, - ) - ) - manager._accumulators = {AccumulatorType.METRIC_RESULTS: metric_accumulator} - manager._metric_record_accumulators = [metric_accumulator] - manager._stream_exporters = {} - - accumulator = MagicMock() - accumulator.compute_efficiency_metrics = MagicMock( - return_value=[ - MetricResult(tag="total_gpu_power", header="h", unit="W", avg=0.0) - ] - ) - manager._gpu_telemetry_accumulator = accumulator - - manager._records_tracker = MagicMock() - manager._records_tracker.create_stats_for_phase.return_value = MagicMock( - start_ns=None, - requests_end_ns=None, - success_records=0, - error_records=0, - ) - manager._error_tracker = MagicMock() - manager._error_tracker.get_error_summary_for_phase.return_value = [] - - result = await manager._process_results(CreditPhase.PROFILING, cancelled=False) - - accumulator.compute_efficiency_metrics.assert_not_called() - manager.warning.assert_called_once() - warning_msg = manager.warning.call_args[0][0] - assert "Skipping efficiency metrics" in warning_msg - assert "start_ns=None" in warning_msg - assert "requests_end_ns=None" in warning_msg - assert {r.tag for r in result.results.records} == {"request_latency"} - - @pytest.mark.asyncio - async def test_partial_none_phase_window_also_skips(self) -> None: - manager = RecordsManager.__new__(RecordsManager) - - manager.debug = MagicMock() - manager.info = MagicMock() - manager.warning = MagicMock() - manager.error = MagicMock() - manager.exception = MagicMock() - manager.service_id = "records-manager-test" - manager._latest_branch_stats = None - manager.publish = AsyncMock() - - manager.run = MagicMock() - manager.run.cfg.gpu_telemetry_disabled = True - manager.run.cfg.server_metrics_disabled = True - manager.run.cfg.network_latency.enabled = False - - metric_accumulator = MagicMock() - metric_accumulator.summarize = AsyncMock( - return_value=AccumulatorMetricsSummary(results={}) - ) - manager._accumulators = {AccumulatorType.METRIC_RESULTS: metric_accumulator} - manager._metric_record_accumulators = [metric_accumulator] - manager._stream_exporters = {} - - accumulator = MagicMock() - manager._gpu_telemetry_accumulator = accumulator - - manager._records_tracker = MagicMock() - manager._records_tracker.create_stats_for_phase.return_value = MagicMock( - start_ns=1_000_000_000, - requests_end_ns=None, - success_records=0, - error_records=0, - ) - manager._error_tracker = MagicMock() - manager._error_tracker.get_error_summary_for_phase.return_value = [] - - await manager._process_results(CreditPhase.PROFILING, cancelled=False) - - accumulator.compute_efficiency_metrics.assert_not_called() - manager.warning.assert_called_once() + stub_analyzer.analyze.assert_awaited_once() class TestMidRunCacheReportingHint: diff --git a/tests/unit/records/test_records_manager_process_results.py b/tests/unit/records/test_records_manager_process_results.py index 30bfa3ffba..f55cc7d127 100644 --- a/tests/unit/records/test_records_manager_process_results.py +++ b/tests/unit/records/test_records_manager_process_results.py @@ -169,9 +169,8 @@ def _make_manager_mock( mgr._bucket_accumulator_summary = ( RecordsManager._bucket_accumulator_summary.__get__(mgr) ) - mgr._apply_gpu_efficiency_metrics = ( - RecordsManager._apply_gpu_efficiency_metrics.__get__(mgr) - ) + mgr._analyzers = [] + mgr._run_analyzers = RecordsManager._run_analyzers.__get__(mgr) mgr._finalize_stream_exporters = RecordsManager._finalize_stream_exporters.__get__( mgr ) diff --git a/tests/unit/server_metrics/test_accumulator.py b/tests/unit/server_metrics/test_accumulator.py index 25a2ebfc4e..e30bebed93 100644 --- a/tests/unit/server_metrics/test_accumulator.py +++ b/tests/unit/server_metrics/test_accumulator.py @@ -4,6 +4,7 @@ import pytest +from aiperf.common.accumulator_protocols import ExportContext from aiperf.common.enums import PrometheusMetricType from aiperf.common.models.error_models import ErrorDetailsCount from aiperf.common.models.server_metrics_models import ( @@ -106,8 +107,10 @@ async def test_export_results_no_data(self, mock_cfg: BenchmarkRun) -> None: processor = ServerMetricsAccumulator(mock_cfg) result = await processor.export_results( - start_ns=1_000_000_000, - end_ns=2_000_000_000, + ExportContext( + start_ns=1_000_000_000, + end_ns=2_000_000_000, + ) ) assert result is None @@ -136,7 +139,9 @@ async def test_export_results_with_data( start_ns = 1_000_000_000 end_ns = 2_000_000_000 - result = await processor.export_results(start_ns=start_ns, end_ns=end_ns) + result = await processor.export_results( + ExportContext(start_ns=start_ns, end_ns=end_ns) + ) assert result is not None assert isinstance(result, ServerMetricsResults) @@ -173,10 +178,12 @@ async def test_export_results_includes_warmup_endpoint_summaries( await processor.process_server_metrics_record(record) result = await processor.export_results( - start_ns=2_000_000_000, - end_ns=3_000_000_000, - warmup_start_ns=1_000_000_000, - warmup_end_ns=2_000_000_000, + ExportContext( + start_ns=2_000_000_000, + end_ns=3_000_000_000, + warmup_start_ns=1_000_000_000, + warmup_end_ns=2_000_000_000, + ) ) assert result is not None @@ -225,10 +232,12 @@ async def test_export_results_degenerate_warmup_window_preserves_profiling( await processor.process_server_metrics_record(record) result = await processor.export_results( - start_ns=2_000_000_000, - end_ns=3_000_000_000, - warmup_start_ns=1_000_000_000, - warmup_end_ns=1_000_000_000, # degenerate: start == end + ExportContext( + start_ns=2_000_000_000, + end_ns=3_000_000_000, + warmup_start_ns=1_000_000_000, + warmup_end_ns=1_000_000_000, # degenerate: start == end + ) ) assert result is not None @@ -272,8 +281,10 @@ async def test_export_results_degenerate_profiling_window_does_not_raise( # start_ns == end_ns == last_update_ns => export_end_ns collapses to # start_ns, a degenerate window for the eager parquet TimeRangeFilter. result = await processor.export_results( - start_ns=1_000_000_000, - end_ns=1_000_000_000, + ExportContext( + start_ns=1_000_000_000, + end_ns=1_000_000_000, + ) ) assert result is not None @@ -301,9 +312,11 @@ async def test_export_results_with_error_summary( ] result = await processor.export_results( - start_ns=1_000_000_000, - end_ns=2_000_000_000, - error_summary=error_summary, + ExportContext( + start_ns=1_000_000_000, + end_ns=2_000_000_000, + error_summary=error_summary, + ) ) assert result is not None @@ -334,8 +347,10 @@ async def test_export_results_with_time_filter( # export_results now constructs per-endpoint TimeFilters internally # start_ns and end_ns define the profiling phase bounds result = await processor.export_results( - start_ns=1_000_000_000, # Profiling start - end_ns=2_000_000_000, # Profiling end + ExportContext( + start_ns=1_000_000_000, # Profiling start + end_ns=2_000_000_000, # Profiling end + ) ) assert result is not None @@ -367,8 +382,10 @@ async def test_export_results_multiple_endpoints( await processor.process_server_metrics_record(record) result = await processor.export_results( - start_ns=1_000_000_000, - end_ns=2_000_000_000, + ExportContext( + start_ns=1_000_000_000, + end_ns=2_000_000_000, + ) ) assert result is not None @@ -402,8 +419,10 @@ async def test_export_results_with_labeled_metrics( await processor.process_server_metrics_record(record) result = await processor.export_results( - start_ns=1_000_000_000, - end_ns=2_000_000_000, + ExportContext( + start_ns=1_000_000_000, + end_ns=2_000_000_000, + ) ) assert result is not None @@ -435,8 +454,10 @@ async def test_export_results_computes_endpoint_metadata( await processor.process_server_metrics_record(record) result = await processor.export_results( - start_ns=1_000_000_000, - end_ns=6_000_000_000, + ExportContext( + start_ns=1_000_000_000, + end_ns=6_000_000_000, + ) ) assert result is not None @@ -486,8 +507,10 @@ async def test_export_results_median_robust_to_outliers( await processor.process_server_metrics_record(record) result = await processor.export_results( - start_ns=1_000_000_000, - end_ns=10_000_000_000, + ExportContext( + start_ns=1_000_000_000, + end_ns=10_000_000_000, + ) ) summary = list(result.endpoint_summaries.values())[0] @@ -532,8 +555,10 @@ async def test_slice_duration_controls_window_size(self): await processor.process_server_metrics_record(record) result = await processor.export_results( - start_ns=0, - end_ns=9_000_000_000, + ExportContext( + start_ns=0, + end_ns=9_000_000_000, + ) ) assert result is not None diff --git a/tests/unit/server_metrics/test_accumulator_sglang_residuals.py b/tests/unit/server_metrics/test_accumulator_sglang_residuals.py index c60950080a..1dd57b0083 100644 --- a/tests/unit/server_metrics/test_accumulator_sglang_residuals.py +++ b/tests/unit/server_metrics/test_accumulator_sglang_residuals.py @@ -22,6 +22,7 @@ import pytest +from aiperf.common.accumulator_protocols import ExportContext from aiperf.common.enums import PrometheusMetricType from aiperf.common.models.server_metrics_models import ( MetricFamily, @@ -209,8 +210,10 @@ async def capture_export_filter(time_filter): ) result = await acc.export_results( - start_ns=1_000_000_000, - end_ns=2_000_000_000, + ExportContext( + start_ns=1_000_000_000, + end_ns=2_000_000_000, + ) ) assert result is not None diff --git a/tests/unit/server_metrics/test_unknown_metrics.py b/tests/unit/server_metrics/test_unknown_metrics.py index a06e8e42f0..6886ad8a4b 100644 --- a/tests/unit/server_metrics/test_unknown_metrics.py +++ b/tests/unit/server_metrics/test_unknown_metrics.py @@ -32,6 +32,7 @@ import pytest from aiperf_mock_server.node_exporter_faker import NodeExporterFaker +from aiperf.common.accumulator_protocols import ExportContext from aiperf.common.enums import PrometheusMetricType from aiperf.common.models.server_metrics_models import ( CounterMetricData, @@ -196,7 +197,7 @@ async def test_endpoint_summary_uses_unknown_metric_data( await proc.process_server_metrics_record( _untyped_record("node_netstat_Icmp_InErrors", v, timestamp_ns=i + 1) ) - results = await proc.export_results(start_ns=0, end_ns=10) + results = await proc.export_results(ExportContext(start_ns=0, end_ns=10)) assert results is not None (summary,) = results.endpoint_summaries.values() metric = summary.metrics["node_netstat_Icmp_InErrors"] From 576d1c38692ccf15cce22e33f66966f503e3ceab Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 19:34:48 -0700 Subject: [PATCH 10/39] feat(metrics): fold power-integration energy source into energy analyzer Port the compute refinements from the k8s-post-port-reflow energy analyzer into EnergyEfficiencyAnalyzer: - Add EnergySource detection: prefer the DCGM energy_consumption counter (average power = energy / profiling_duration); fall back to power integration (energy = fleet_power * duration) when only the power gauge is exposed (pynvml/amdsmi collectors with no energy counter). Skip the family when neither signal is present. - avg power is now the time-averaged fleet power backing total_gpu_energy, and performance/token per-watt metrics divide by it. - Read output tokens from total_osl (metrics summary) rather than the legacy total_output_tokens scan. - Add two per-watt metrics: output_tps_per_watt (tokens/sec/W) and goodput_per_watt (good-req/s/W, emitted only when goodput SLOs are set). Keeps the branch-native additions (energy_delay_product, energy_per_user). Verified end-to-end against the mock DCGM faker (11 metrics emitted, formulas cross-checked exactly against summary inputs). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/metrics-reference.md | 59 ++++++-- src/aiperf/common/enums/metric_enums.py | 2 + .../metrics/energy_efficiency_analyzer.py | 133 ++++++++++++------ .../metrics/types/power_efficiency_metrics.py | 27 +++- .../test_energy_efficiency_analyzer.py | 63 +++++++-- 5 files changed, 220 insertions(+), 64 deletions(-) diff --git a/docs/metrics-reference.md b/docs/metrics-reference.md index f316c5eac0..b537e1a491 100644 --- a/docs/metrics-reference.md +++ b/docs/metrics-reference.md @@ -129,6 +129,8 @@ This document provides a comprehensive reference of all metrics available in AIP - [Energy per Total Token](#energy-per-total-token) - [Energy per Request](#energy-per-request) - [Performance per Watt](#performance-per-watt) + - [Output Tokens per Second per Watt](#output-tokens-per-second-per-watt) + - [Goodput per Watt](#goodput-per-watt) - [Energy Delay Product](#energy-delay-product) - [Metric Flags Reference](#metric-flags-reference) @@ -1878,7 +1880,9 @@ http_req_chunks_received = trace.response_chunks_count > [!NOTE] > All metrics in this section require `--gpu-telemetry` to be enabled and the underlying collector (DCGM, pynvml, or amdsmi) to expose the relevant signal (`gpu_power_usage` and/or `energy_consumption`). They are computed once per profiling phase by the `EnergyEfficiencyAnalyzer` — an `analyzer` plugin that joins the GPU-telemetry accumulator (energy/power) to the metrics-accumulator summary (tokens/throughput/latency) via the `SummaryContext` — not by the standard derivation walk. See the [Externally-Injected Derived Metric pattern](dev/patterns.md#externally-injected-derived-metric-pattern) and the `analyzer` plugin category. -The family is skipped entirely when GPU telemetry is not collected. Each tag is independently omitted when its underlying signal is unavailable (e.g. `energy_per_user` only appears for concurrency runs). Tags: `energy_delay_product`, `performance_per_watt`, `average_gpu_power`, `total_gpu_energy`, `total_gpu_power`, `energy_per_total_token`, `energy_per_output_token`, `energy_per_request`, `output_tokens_per_joule`, `energy_per_user`. +**Energy source.** Total GPU energy is taken from the DCGM `energy_consumption` counter delta when available (`source = dcgm_counter`), and the time-averaged fleet power is derived from it as `total_gpu_energy / profiling_duration_s`. When no energy counter is exposed (e.g. pynvml/amdsmi power-only collectors), the analyzer falls back to **power integration** (`source = power_integration`): `total_gpu_energy = total_gpu_power * profiling_duration_s`, and the average power is the fleet power gauge itself. If neither signal is present the family is skipped. + +The family is skipped entirely when GPU telemetry is not collected. Each tag is independently omitted when its underlying signal is unavailable (e.g. `energy_per_user` only appears for concurrency runs; `goodput_per_watt` only when goodput SLOs are configured). Tags: `energy_delay_product`, `performance_per_watt`, `output_tps_per_watt`, `goodput_per_watt`, `average_gpu_power`, `total_gpu_energy`, `total_gpu_power`, `energy_per_total_token`, `energy_per_output_token`, `energy_per_request`, `output_tokens_per_joule`, `energy_per_user`. ### Total GPU Power @@ -1939,14 +1943,14 @@ Inference energy efficiency: number of output tokens produced per joule of GPU e **Formula:** ```python -output_tokens_per_joule = total_output_tokens / total_gpu_energy +output_tokens_per_joule = total_osl / total_gpu_energy ``` **Notes:** - Unit: `tokens/J`. - Flagged `LARGER_IS_BETTER | PRODUCES_TOKENS_ONLY`. -- Numerator comes from the request records (`total_output_tokens`); denominator comes from the GPU telemetry counter delta above. The header reports the energy-side GPU count, since that's the cohort the metric depends on. -- Omitted when `total_output_tokens` is absent from the records or aggregate `total_gpu_energy` is zero. +- Numerator is total output tokens (`total_osl` from the metrics summary); denominator comes from the GPU telemetry counter delta (or integrated power) above. +- Omitted when `total_osl` is absent from the summary or aggregate `total_gpu_energy` is zero. --- @@ -1976,11 +1980,14 @@ energy_per_user_j = total_gpu_energy / concurrency **Type:** [Derived Metric](#derived-metrics) (externally injected) -Mean GPU power draw per reporting GPU over the profiling window, in watts (`W`). Complements `total_gpu_power` (the fleet sum) with a per-GPU view. +Time-averaged total GPU power over the profiling window, in watts (`W`) — the fleet power that actually produced `total_gpu_energy`. Under the DCGM counter path this is derived from energy; under power integration it equals `total_gpu_power`. **Formula:** ```python -average_gpu_power_w = total_gpu_power / reporting_gpu_count +# dcgm_counter source: +average_gpu_power_w = total_gpu_energy / profiling_duration_s +# power_integration source: +average_gpu_power_w = total_gpu_power ``` --- @@ -1993,7 +2000,7 @@ The primary energy-efficiency metric: GPU energy spent per output token, in mill **Formula:** ```python -energy_per_output_token_mj = total_gpu_energy * 1000 / total_output_tokens +energy_per_output_token_mj = total_gpu_energy * 1000 / total_osl ``` --- @@ -2006,7 +2013,7 @@ GPU energy per total (input + output) token, in `mJ/token`. Captures combined pr **Formula:** ```python -energy_per_total_token_mj = total_gpu_energy * 1000 / (total_isl + total_output_tokens) +energy_per_total_token_mj = total_gpu_energy * 1000 / (total_isl + total_osl) ``` --- @@ -2032,9 +2039,43 @@ Request throughput per watt of GPU power draw, in `requests/sec/W`. Higher is be **Formula:** ```python -performance_per_watt = request_throughput / total_gpu_power +performance_per_watt = request_throughput / average_gpu_power +``` + +--- + +### Output Tokens per Second per Watt + +**Type:** [Derived Metric](#derived-metrics) (externally injected) + +Output-token throughput per watt of GPU power draw, in `tokens/sec/W`. Higher is better — the token-level companion to `performance_per_watt`, insensitive to request size. + +**Formula:** +```python +output_tps_per_watt = output_token_throughput / average_gpu_power +``` + +**Notes:** +- Flagged `LARGER_IS_BETTER | PRODUCES_TOKENS_ONLY`. +- Omitted when `output_token_throughput` is absent or average power is zero. + +--- + +### Goodput per Watt + +**Type:** [Derived Metric](#derived-metrics) (externally injected) + +SLO-passing request throughput (goodput) per watt of GPU power draw, in `good-req/s/W`. Higher is better — measures efficient work that actually met its latency SLOs, not raw throughput. + +**Formula:** +```python +goodput_per_watt = goodput / average_gpu_power ``` +**Notes:** +- Flagged `LARGER_IS_BETTER`. +- Only meaningful when goodput SLOs are configured (`--goodput`); omitted otherwise or when average power is zero. + --- ### Energy Delay Product diff --git a/src/aiperf/common/enums/metric_enums.py b/src/aiperf/common/enums/metric_enums.py index 3d34f1cc3e..8e0ec7ef60 100644 --- a/src/aiperf/common/enums/metric_enums.py +++ b/src/aiperf/common/enums/metric_enums.py @@ -190,6 +190,7 @@ class GenericMetricUnit(BaseMetricUnit): ERRORS = _unit("errors") IMAGE = _unit("image") IMAGES = _unit("images") + GOODPUT_PER_WATT = _unit("good-req/s/W") JOULES_PER_REQUEST = _unit("joules/request") JOULES_PER_USER = _unit("joules/user") JOULE_SECONDS = _unit("J*s") @@ -199,6 +200,7 @@ class GenericMetricUnit(BaseMetricUnit): REQUESTS = _unit("requests") REQUESTS_PER_SECOND_PER_WATT = _unit("requests/sec/W") TOKENS = _unit("tokens") + TOKENS_PER_SECOND_PER_WATT = _unit("tokens/sec/W") TOKENS_PER_JOULE = _unit("tokens/J") USER = _unit("user") USERS = _unit("users") diff --git a/src/aiperf/metrics/energy_efficiency_analyzer.py b/src/aiperf/metrics/energy_efficiency_analyzer.py index 7bccb76521..c60a14a38b 100644 --- a/src/aiperf/metrics/energy_efficiency_analyzer.py +++ b/src/aiperf/metrics/energy_efficiency_analyzer.py @@ -7,12 +7,18 @@ off the ``MetricsAccumulator`` summary) to emit the energy-efficiency metric family. Runs at summarize time via the SummaryContext; skipped by RecordsManager when GPU telemetry is not collected. See design doc ``0005-energy-efficiency-metrics.md``. + +Energy source: prefers the DCGM ``energy_consumption`` counter delta; falls back +to power-integration (``fleet_power * duration``) when only the power gauge is +available (e.g. pynvml/amdsmi collectors that expose no energy counter). """ from __future__ import annotations +import enum from typing import TYPE_CHECKING, Any +from aiperf.common.constants import NANOS_PER_SECOND from aiperf.common.logging import AIPerfLogger from aiperf.common.models import MetricResult from aiperf.metrics.types.power_efficiency_metrics import ( @@ -22,7 +28,9 @@ EnergyPerRequestMetric, EnergyPerTotalTokenMetric, EnergyPerUserMetric, + GoodputPerWattMetric, OutputTokensPerJouleMetric, + OutputTokensPerSecondPerWattMetric, PerformancePerWattMetric, TotalGpuEnergyMetric, TotalGpuPowerMetric, @@ -40,6 +48,14 @@ _MS_PER_SECOND = 1000.0 +class EnergySource(str, enum.Enum): + """How total GPU energy was determined for this run.""" + + DCGM_COUNTER = "dcgm_counter" + POWER_INTEGRATION = "power_integration" + UNAVAILABLE = "unavailable" + + def _result(metric_cls: type, value: float) -> MetricResult: """Build the injected MetricResult for an energy metric class.""" return MetricResult( @@ -57,8 +73,7 @@ class EnergyEfficiencyAnalyzer: Reads the live ``GPUTelemetryAccumulator`` (windowed energy/power) and the ``MetricsAccumulator`` summary (token/throughput/latency totals) from the SummaryContext, then computes the energy metric family. Each metric is - emitted only when its inputs are available, mirroring the per-signal omission - of the pipeline it replaces. + emitted only when its inputs are available. """ def __init__( @@ -95,71 +110,92 @@ def metric(tag: str) -> float | None: start_ns = ctx.start_ns or None end_ns = ctx.end_ns or None - energy_j, energy_count = gpu.total_energy_joules(start_ns, end_ns) - power_w, power_count = gpu.total_power_watts(start_ns, end_ns) - - out = self._power_metrics(power_w, power_count) - out += self._energy_metrics(energy_j, energy_count, metric) - if ( - power_count > 0 - and power_w > 0 - and (throughput := metric("request_throughput")) - ): - out.append(_result(PerformancePerWattMetric, throughput / power_w)) + duration_s = ( + (ctx.end_ns - ctx.start_ns) / NANOS_PER_SECOND + if ctx.end_ns > ctx.start_ns + else 0.0 + ) + energy = gpu.total_energy_joules(start_ns, end_ns) + power = gpu.total_power_watts(start_ns, end_ns) + total_energy_j, avg_power_w, source = self._resolve_energy( + energy, power, duration_s + ) + + out: list[MetricResult] = [] + if power[1] > 0: + out.append(_result(TotalGpuPowerMetric, power[0])) + if source is EnergySource.UNAVAILABLE or total_energy_j <= 0: + return out + + out.append(_result(AverageGpuPowerMetric, avg_power_w)) + out.append(_result(TotalGpuEnergyMetric, total_energy_j)) + out += self._energy_ratio_metrics(total_energy_j, metric, self._concurrency()) + out += self._per_watt_metrics(avg_power_w, metric) _logger.debug( lambda: ( f"EnergyEfficiencyAnalyzer emitted {len(out)} metrics " - f"(energy={energy_j:.2f}J/{energy_count}gpu, " - f"power={power_w:.2f}W/{power_count}gpu)" + f"(source={source.value}, energy={total_energy_j:.2f}J, " + f"avg_power={avg_power_w:.2f}W)" ) ) return out @staticmethod - def _power_metrics(power_w: float, power_count: int) -> list[MetricResult]: - if power_count <= 0: - return [] - return [ - _result(TotalGpuPowerMetric, power_w), - _result(AverageGpuPowerMetric, power_w / power_count), - ] - - def _energy_metrics( + def _resolve_energy( + energy: tuple[float, int], + power: tuple[float, int], + duration_s: float, + ) -> tuple[float, float, EnergySource]: + """Total energy (J) + average fleet power (W) + how they were determined. + + ``energy``/``power`` are ``(value, gpu_count)`` from the telemetry + accumulator. Prefers the DCGM energy counter (average power = energy / + duration); falls back to power-integration (energy = fleet_power * + duration, average power = fleet_power) when only the power gauge is + available. + """ + energy_j, energy_count = energy + power_w, power_count = power + if energy_count > 0 and energy_j > 0: + avg = energy_j / duration_s if duration_s > 0 else 0.0 + return energy_j, avg, EnergySource.DCGM_COUNTER + if power_count > 0 and power_w > 0 and duration_s > 0: + return power_w * duration_s, power_w, EnergySource.POWER_INTEGRATION + return 0.0, 0.0, EnergySource.UNAVAILABLE + + def _energy_ratio_metrics( self, - energy_j: float, - energy_count: int, + total_energy_j: float, metric: Callable[[str], float | None], + concurrency: int | None, ) -> list[MetricResult]: - if energy_count <= 0 or energy_j <= 0: - return [] - out: list[MetricResult] = [_result(TotalGpuEnergyMetric, energy_j)] - - output_tokens = metric("total_output_tokens") + out: list[MetricResult] = [] + output_tokens = metric("total_osl") if output_tokens: - out.append(_result(OutputTokensPerJouleMetric, output_tokens / energy_j)) + out.append( + _result(OutputTokensPerJouleMetric, output_tokens / total_energy_j) + ) out.append( _result( EnergyPerOutputTokenMetric, - energy_j * _MS_PER_SECOND / output_tokens, + total_energy_j * _MS_PER_SECOND / output_tokens, ) ) total_tokens = (metric("total_isl") or 0.0) + (output_tokens or 0.0) if total_tokens > 0: out.append( _result( - EnergyPerTotalTokenMetric, energy_j * _MS_PER_SECOND / total_tokens + EnergyPerTotalTokenMetric, + total_energy_j * _MS_PER_SECOND / total_tokens, ) ) - energy_per_request_j = None if request_count := metric("request_count"): - energy_per_request_j = energy_j / request_count + energy_per_request_j = total_energy_j / request_count out.append(_result(EnergyPerRequestMetric, energy_per_request_j)) - if (concurrency := self._concurrency()) is not None: - out.append(_result(EnergyPerUserMetric, energy_j / concurrency)) - - # Energy-delay product: J/request * mean request latency (s). + if concurrency is not None: + out.append(_result(EnergyPerUserMetric, total_energy_j / concurrency)) latency_ms = metric("request_latency") if energy_per_request_j is not None and latency_ms: out.append( @@ -169,3 +205,20 @@ def _energy_metrics( ) ) return out + + @staticmethod + def _per_watt_metrics( + avg_power_w: float, metric: Callable[[str], float | None] + ) -> list[MetricResult]: + if avg_power_w <= 0: + return [] + out: list[MetricResult] = [] + if throughput := metric("request_throughput"): + out.append(_result(PerformancePerWattMetric, throughput / avg_power_w)) + if output_tps := metric("output_token_throughput"): + out.append( + _result(OutputTokensPerSecondPerWattMetric, output_tps / avg_power_w) + ) + if goodput := metric("goodput"): + out.append(_result(GoodputPerWattMetric, goodput / avg_power_w)) + return out diff --git a/src/aiperf/metrics/types/power_efficiency_metrics.py b/src/aiperf/metrics/types/power_efficiency_metrics.py index 6d22c494cf..c19ef65305 100644 --- a/src/aiperf/metrics/types/power_efficiency_metrics.py +++ b/src/aiperf/metrics/types/power_efficiency_metrics.py @@ -70,8 +70,33 @@ class PerformancePerWattMetric(_InjectedEnergyMetric): flags = MetricFlags.LARGER_IS_BETTER +class OutputTokensPerSecondPerWattMetric(_InjectedEnergyMetric): + """Output-token throughput per watt of GPU power draw (tokens/sec/W).""" + + __is_abstract__ = False + tag = "output_tps_per_watt" + header = "Output Tokens per Second per Watt" + unit = GenericMetricUnit.TOKENS_PER_SECOND_PER_WATT + display_order = 712 + flags = MetricFlags.LARGER_IS_BETTER | MetricFlags.PRODUCES_TOKENS_ONLY + + +class GoodputPerWattMetric(_InjectedEnergyMetric): + """Goodput (SLO-passing requests/sec) per watt of GPU power draw (good-req/s/W). + + Only meaningful when goodput SLOs are configured; omitted otherwise. + """ + + __is_abstract__ = False + tag = "goodput_per_watt" + header = "Goodput per Watt" + unit = GenericMetricUnit.GOODPUT_PER_WATT + display_order = 714 + flags = MetricFlags.LARGER_IS_BETTER + + class AverageGpuPowerMetric(_InjectedEnergyMetric): - """Mean GPU power draw per GPU over the profiling window (W).""" + """Time-averaged total GPU power over the profiling window (energy / duration) (W).""" __is_abstract__ = False tag = "average_gpu_power" diff --git a/tests/unit/metrics/test_energy_efficiency_analyzer.py b/tests/unit/metrics/test_energy_efficiency_analyzer.py index 568449f3d4..b84febd413 100644 --- a/tests/unit/metrics/test_energy_efficiency_analyzer.py +++ b/tests/unit/metrics/test_energy_efficiency_analyzer.py @@ -16,9 +16,17 @@ from aiperf.common.accumulator_protocols import SummaryContext from aiperf.common.models import MetricResult from aiperf.metrics.accumulator_models import AccumulatorMetricsSummary -from aiperf.metrics.energy_efficiency_analyzer import EnergyEfficiencyAnalyzer +from aiperf.metrics.energy_efficiency_analyzer import ( + EnergyEfficiencyAnalyzer, + EnergySource, +) from aiperf.plugin.enums import AccumulatorType +# 10-second profiling window in nanoseconds (round numbers for the formulas). +_START_NS = 0 +_END_NS = 10 * 1_000_000_000 +_DURATION_S = 10.0 + class _StubGpu: """Minimal GPUTelemetryAccumulator query surface for the analyzer.""" @@ -57,8 +65,8 @@ def _ctx(gpu, summary) -> SummaryContext: accumulator_outputs=( {AccumulatorType.METRIC_RESULTS: summary} if summary else {} ), - start_ns=1_000, - end_ns=2_000, + start_ns=_START_NS, + end_ns=_END_NS, ) @@ -67,10 +75,12 @@ class TestEnergyEfficiencyAnalyzer: async def test_full_metric_set_matches_doc_formulas(self) -> None: gpu = _StubGpu(energy=(1000.0, 2), power=(200.0, 2)) summary = _metrics_summary( - total_output_tokens=5000.0, + total_osl=5000.0, total_isl=3000.0, request_count=100.0, request_throughput=10.0, + output_token_throughput=50.0, + goodput=8.0, request_latency=500.0, # ms ) @@ -78,7 +88,8 @@ async def test_full_metric_set_matches_doc_formulas(self) -> None: by_tag = {r.tag: r.avg for r in results} assert by_tag["total_gpu_power"] == pytest.approx(200.0) - assert by_tag["average_gpu_power"] == pytest.approx(100.0) # 200 / 2 GPUs + # DCGM counter path: avg power = energy / duration = 1000 / 10s. + assert by_tag["average_gpu_power"] == pytest.approx(100.0) assert by_tag["total_gpu_energy"] == pytest.approx(1000.0) assert by_tag["output_tokens_per_joule"] == pytest.approx(5.0) # 5000 / 1000 assert by_tag["energy_per_output_token"] == pytest.approx( @@ -90,11 +101,13 @@ async def test_full_metric_set_matches_doc_formulas(self) -> None: assert by_tag["energy_per_request"] == pytest.approx(10.0) # 1000 / 100 assert by_tag["energy_per_user"] == pytest.approx(125.0) # 1000 / 8 assert by_tag["energy_delay_product"] == pytest.approx(5.0) # 10 * 0.5s - assert by_tag["performance_per_watt"] == pytest.approx(0.05) # 10 / 200 + assert by_tag["performance_per_watt"] == pytest.approx(0.1) # 10 / 100 + assert by_tag["output_tps_per_watt"] == pytest.approx(0.5) # 50 / 100 + assert by_tag["goodput_per_watt"] == pytest.approx(0.08) # 8 / 100 @pytest.mark.asyncio async def test_no_gpu_accumulator_returns_empty(self) -> None: - summary = _metrics_summary(total_output_tokens=5000.0) + summary = _metrics_summary(total_osl=5000.0) results = await _analyzer().analyze(_ctx(None, summary)) assert results == [] @@ -105,23 +118,45 @@ async def test_no_metrics_summary_returns_empty(self) -> None: assert results == [] @pytest.mark.asyncio - async def test_missing_signals_omit_dependent_metrics(self) -> None: - # No energy signal (count 0) and no tokens: only power-derived metrics. + async def test_power_integration_fallback_when_no_energy_counter(self) -> None: + # No energy counter (count 0); fall back to power * duration. gpu = _StubGpu(energy=(0.0, 0), power=(150.0, 3)) summary = _metrics_summary(request_throughput=6.0) results = await _analyzer(concurrency=None).analyze(_ctx(gpu, summary)) by_tag = {r.tag: r.avg for r in results} assert by_tag["total_gpu_power"] == pytest.approx(150.0) - assert by_tag["average_gpu_power"] == pytest.approx(50.0) + # Integrated energy = 150 W * 10 s = 1500 J; avg power = fleet power. + assert by_tag["total_gpu_energy"] == pytest.approx(1500.0) + assert by_tag["average_gpu_power"] == pytest.approx(150.0) assert by_tag["performance_per_watt"] == pytest.approx(0.04) # 6 / 150 - assert "total_gpu_energy" not in by_tag - assert "energy_per_output_token" not in by_tag - assert "energy_per_user" not in by_tag + assert "energy_per_output_token" not in by_tag # no tokens + assert "energy_per_user" not in by_tag # concurrency None + + @pytest.mark.asyncio + async def test_no_energy_signal_at_all_returns_empty(self) -> None: + # Neither energy counter nor power gauge: source UNAVAILABLE, nothing emitted. + gpu = _StubGpu(energy=(0.0, 0), power=(0.0, 0)) + summary = _metrics_summary(request_throughput=6.0) + results = await _analyzer().analyze(_ctx(gpu, summary)) + assert results == [] + + @pytest.mark.asyncio + async def test_resolve_energy_source_detection(self) -> None: + _, _, counter = EnergyEfficiencyAnalyzer._resolve_energy( + (1000.0, 2), (200.0, 2), _DURATION_S + ) + assert counter is EnergySource.DCGM_COUNTER + _, _, integ = EnergyEfficiencyAnalyzer._resolve_energy( + (0.0, 0), (150.0, 3), _DURATION_S + ) + assert integ is EnergySource.POWER_INTEGRATION + _, _, none = EnergyEfficiencyAnalyzer._resolve_energy((0.0, 0), (0.0, 0), 0.0) + assert none is EnergySource.UNAVAILABLE @pytest.mark.asyncio async def test_concurrency_gates_energy_per_user(self) -> None: gpu = _StubGpu(energy=(1000.0, 1), power=(100.0, 1)) - summary = _metrics_summary(total_output_tokens=1000.0) + summary = _metrics_summary(total_osl=1000.0) results = await _analyzer(concurrency=None).analyze(_ctx(gpu, summary)) assert "energy_per_user" not in {r.tag for r in results} From 84681405408f3a1838a2db4a5e94df11464cdc6b Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 19:56:39 -0700 Subject: [PATCH 11/39] feat(accuracy): add accuracy record/summary models, messages, and enums Introduce the data models, wire messages, and MessageType enum values for a dedicated "accuracy" record-type channel (mirroring gpu_telemetry and server_metrics), replacing the shared metric_records shoehorn in a later phase. - AccuracyRecordsData (record_type ClassVar), TaskAccuracyStats, AccuracySummary (to_json/to_csv), ProcessAccuracyResult in accuracy/models.py - AccuracyRecordsMessage, ProcessAccuracyResultMessage in accuracy_messages.py - MessageType.ACCURACY_RECORD / PROCESS_ACCURACY_RESULT - Nothing wires/routes these yet (later tasks) Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/accuracy/models.py | 137 +++++++++++++- src/aiperf/common/enums/enums.py | 2 + src/aiperf/common/messages/__init__.py | 6 + .../common/messages/accuracy_messages.py | 38 ++++ tests/unit/accuracy/test_accuracy_models.py | 174 ++++++++++++++++++ 5 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 src/aiperf/common/messages/accuracy_messages.py create mode 100644 tests/unit/accuracy/test_accuracy_models.py diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index f0ce40f914..a4e2782b71 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -1,11 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Any, Literal +from __future__ import annotations + +from typing import Any, ClassVar, Literal from pydantic import Field from typing_extensions import TypedDict +from aiperf.common.enums import CreditPhase from aiperf.common.models.base_models import AIPerfBaseModel # Summary/metric tags (the ``accuracy.`` dot namespace) emitted by @@ -84,3 +87,135 @@ class BenchmarkProblem(AIPerfBaseModel): "Turn.raw_messages because that field also accepts tool-call and multi-modal " "messages from other callers (e.g. MooncakeTrace).", ) + + +class AccuracyRecordsData(AIPerfBaseModel): + """Per-graded-response record that flows on the dedicated ``accuracy`` channel. + + Mirrors the record-type pattern used by ``ServerMetricsRecord`` and + ``TelemetryRecord``: ``record_type`` is a plain ``ClassVar`` (not a Pydantic + field) read by the routing layer via ``getattr(record, "record_type")``. + """ + + record_type: ClassVar[str] = "accuracy" + + session_num: int = Field( + description="Conversation/session index this response came from, used to map to task" + ) + worker_id: str = Field( + description="ID of the record processor that produced this record" + ) + benchmark_phase: CreditPhase = Field( + description="Benchmark phase active when grading completed (warmup vs profiling)" + ) + timestamp_ns: int = Field( + description="Nanosecond wall-clock timestamp when grading completed" + ) + task: str | None = Field( + default=None, + description="Accuracy task/subtask name (e.g. an MMLU subtask); " + "None when the dataset has no task label", + ) + grader_name: str = Field(description="Which grader scored this response") + passed: bool = Field( + description="Whether the response was graded correct (maps from GradingResult.correct)" + ) + unparsed: bool = Field( + default=False, + description="Whether the model output needed a regex fallback " + "(maps from GradingResult.unparsed)", + ) + confidence: float = Field(ge=0, le=1, description="Grading confidence (0.0 to 1.0)") + expected: str = Field( + description="Ground-truth answer (maps from GradingResult.ground_truth)" + ) + actual: str = Field( + description="Answer extracted from the model response " + "(maps from GradingResult.extracted_answer)" + ) + reasoning: str = Field(description="Grader's explanation of the grading decision") + + +class TaskAccuracyStats(AIPerfBaseModel): + """Per-task accuracy rollup.""" + + total: int = Field(description="Total responses evaluated for this task") + passed: int = Field(description="Number graded correct for this task") + unparsed: int = Field( + description="Number that needed a regex fallback for this task" + ) + accuracy_rate: float = Field( + description="passed/total for this task, 0.0 when total==0" + ) + unparsed_rate: float = Field( + description="unparsed/total for this task, 0.0 when total==0" + ) + + +class AccuracySummary(AIPerfBaseModel): + """Accumulator result payload; structured replacement for today's list[MetricResult].""" + + total_evaluated: int = Field( + description="Total responses evaluated across all tasks" + ) + total_passed: int = Field( + description="Total responses graded correct across all tasks" + ) + accuracy_rate: float = Field( + description="total_passed/total_evaluated, 0.0 when total_evaluated==0" + ) + overall_unparsed: int = Field( + description="Total responses that needed a regex fallback across all tasks" + ) + grader_name: str | None = Field( + default=None, description="Grader that scored these responses, if uniform" + ) + per_task: dict[str, TaskAccuracyStats] = Field( + default_factory=dict, description="Per-task accuracy rollups keyed by task name" + ) + + def to_json(self) -> dict[str, Any]: + """Return a plain-dict representation of the summary.""" + return self.model_dump() + + def to_csv(self) -> list[dict[str, Any]]: + """Return one row per task (sorted by name) plus a trailing OVERALL row. + + Columns: task, total, passed, unparsed, accuracy_rate, unparsed_rate. + Mirrors today's ``accuracy_results.csv`` shape (per-task rows + overall). + """ + rows: list[dict[str, Any]] = [ + { + "task": task, + "total": stats.total, + "passed": stats.passed, + "unparsed": stats.unparsed, + "accuracy_rate": stats.accuracy_rate, + "unparsed_rate": stats.unparsed_rate, + } + for task, stats in sorted(self.per_task.items()) + ] + overall_unparsed_rate = ( + self.overall_unparsed / self.total_evaluated + if self.total_evaluated + else 0.0 + ) + rows.append( + { + "task": "OVERALL", + "total": self.total_evaluated, + "passed": self.total_passed, + "unparsed": self.overall_unparsed, + "accuracy_rate": self.accuracy_rate, + "unparsed_rate": overall_unparsed_rate, + } + ) + return rows + + +class ProcessAccuracyResult(AIPerfBaseModel): + """Wire wrapper for a processed accuracy summary - mirrors ProcessServerMetricsResult.""" + + results: AccuracySummary | None = Field( + default=None, description="The processed accuracy summary" + ) diff --git a/src/aiperf/common/enums/enums.py b/src/aiperf/common/enums/enums.py index 8af6a34a07..32f6874aa8 100644 --- a/src/aiperf/common/enums/enums.py +++ b/src/aiperf/common/enums/enums.py @@ -368,6 +368,8 @@ class MessageType(CaseInsensitiveStrEnum): PROCESS_RECORDS_RESULT = "process_records_result" PROCESS_TELEMETRY_RESULT = "process_telemetry_result" PROCESS_SERVER_METRICS_RESULT = "process_server_metrics_result" + PROCESS_ACCURACY_RESULT = "process_accuracy_result" + ACCURACY_RECORD = "accuracy_record" PROCESS_ALL_RESULTS = "process_all_results" PROFILE_PROGRESS = "profile_progress" PROFILE_RESULTS = "profile_results" diff --git a/src/aiperf/common/messages/__init__.py b/src/aiperf/common/messages/__init__.py index 9c3465ef2a..7fe39db898 100644 --- a/src/aiperf/common/messages/__init__.py +++ b/src/aiperf/common/messages/__init__.py @@ -1,6 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from aiperf.common.messages.accuracy_messages import ( + AccuracyRecordsMessage, + ProcessAccuracyResultMessage, +) from aiperf.common.messages.base_messages import ( ErrorMessage, Message, @@ -77,6 +81,7 @@ ) __all__ = [ + "AccuracyRecordsMessage", "AllRecordsReceivedMessage", "BaseServiceErrorMessage", "BaseServiceMessage", @@ -105,6 +110,7 @@ "ProcessRecordsResponse", "ProcessAllResultsMessage", "ProcessRecordsResultMessage", + "ProcessAccuracyResultMessage", "ProcessServerMetricsResultMessage", "ProcessTelemetryResultMessage", "ProfileCancelCommand", diff --git a/src/aiperf/common/messages/accuracy_messages.py b/src/aiperf/common/messages/accuracy_messages.py new file mode 100644 index 0000000000..80f441fa1b --- /dev/null +++ b/src/aiperf/common/messages/accuracy_messages.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from pydantic import Field + +from aiperf.accuracy.models import AccuracyRecordsData, ProcessAccuracyResult +from aiperf.common.enums import MessageType +from aiperf.common.messages.service_messages import BaseServiceMessage +from aiperf.common.models import ErrorDetails +from aiperf.common.types import MessageTypeT + + +class AccuracyRecordsMessage(BaseServiceMessage): + """Message from a record processor to the records manager carrying graded + accuracy records on the dedicated ``accuracy`` channel. + """ + + message_type: MessageTypeT = MessageType.ACCURACY_RECORD + + records: list[AccuracyRecordsData] = Field( + default_factory=list, + description="The graded accuracy records produced by the record processor", + ) + error: ErrorDetails | None = Field( + default=None, + description="The error details if grading/record production failed.", + ) + + +class ProcessAccuracyResultMessage(BaseServiceMessage): + """Message containing a processed accuracy summary - mirrors ProcessServerMetricsResultMessage.""" + + message_type: MessageTypeT = MessageType.PROCESS_ACCURACY_RESULT + + accuracy_result: ProcessAccuracyResult = Field( + description="The processed accuracy summary results" + ) diff --git a/tests/unit/accuracy/test_accuracy_models.py b/tests/unit/accuracy/test_accuracy_models.py new file mode 100644 index 0000000000..4e31279400 --- /dev/null +++ b/tests/unit/accuracy/test_accuracy_models.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import pytest +from pytest import param + +from aiperf.accuracy.models import ( + AccuracyRecordsData, + AccuracySummary, + ProcessAccuracyResult, + TaskAccuracyStats, +) +from aiperf.common.enums import CreditPhase, MessageType +from aiperf.common.messages import ( + AccuracyRecordsMessage, + ProcessAccuracyResultMessage, +) + + +def _make_record(**overrides) -> AccuracyRecordsData: + defaults = dict( + session_num=0, + worker_id="worker-1", + benchmark_phase=CreditPhase.PROFILING, + timestamp_ns=123456789, + task="mmlu.astronomy", + grader_name="exact_match", + passed=True, + unparsed=False, + confidence=0.9, + expected="B", + actual="B", + reasoning="matched", + ) + defaults.update(overrides) + return AccuracyRecordsData(**defaults) + + +def test_accuracy_record_type_is_classvar_constant() -> None: + a = _make_record(session_num=1) + b = _make_record(session_num=2) + assert AccuracyRecordsData.record_type == "accuracy" + assert a.record_type == "accuracy" + assert b.record_type == "accuracy" + # record_type is a ClassVar, not a constructor field: it must not be dumped. + assert "record_type" not in a.model_dump() + + +def test_accuracy_record_task_defaults_to_none() -> None: + record = _make_record(task=None) + assert record.task is None + + +def test_accuracy_record_round_trip_preserves_fields() -> None: + record = _make_record() + dumped = record.model_dump() + rebuilt = AccuracyRecordsData(**dumped) + assert rebuilt == record + assert rebuilt.passed is True + assert rebuilt.benchmark_phase == CreditPhase.PROFILING + assert rebuilt.expected == "B" + + +def test_task_accuracy_stats_round_trip() -> None: + stats = TaskAccuracyStats( + total=4, + passed=3, + unparsed=1, + accuracy_rate=0.75, + unparsed_rate=0.25, + ) + assert TaskAccuracyStats(**stats.model_dump()) == stats + + +def _summary() -> AccuracySummary: + return AccuracySummary( + total_evaluated=5, + total_passed=3, + accuracy_rate=0.6, + overall_unparsed=1, + grader_name="exact_match", + per_task={ + "mmlu.b": TaskAccuracyStats( + total=2, passed=2, unparsed=0, accuracy_rate=1.0, unparsed_rate=0.0 + ), + "mmlu.a": TaskAccuracyStats( + total=3, passed=1, unparsed=1, accuracy_rate=1 / 3, unparsed_rate=1 / 3 + ), + }, + ) + + +def test_summary_to_json_contains_keys() -> None: + data = _summary().to_json() + assert isinstance(data, dict) + assert data["accuracy_rate"] == 0.6 + assert "per_task" in data + + +def test_summary_to_csv_per_task_rows_sorted_with_overall_last() -> None: + rows = _summary().to_csv() + tasks = [row["task"] for row in rows] + assert tasks == ["mmlu.a", "mmlu.b", "OVERALL"] + overall = rows[-1] + assert overall["total"] == 5 + assert overall["passed"] == 3 + assert overall["accuracy_rate"] == 0.6 + assert overall["unparsed"] == 1 + first = rows[0] + assert set(first.keys()) == { + "task", + "total", + "passed", + "unparsed", + "accuracy_rate", + "unparsed_rate", + } + + +def test_summary_to_csv_empty_no_zero_division() -> None: + empty = AccuracySummary( + total_evaluated=0, + total_passed=0, + accuracy_rate=0.0, + overall_unparsed=0, + ) + rows = empty.to_csv() + assert rows[-1]["task"] == "OVERALL" + assert rows[-1]["accuracy_rate"] == 0.0 + assert rows[-1]["unparsed_rate"] == 0.0 + + +def test_process_accuracy_result_defaults_none() -> None: + assert ProcessAccuracyResult().results is None + wrapped = ProcessAccuracyResult(results=_summary()) + assert wrapped.results is not None + assert wrapped.results.total_evaluated == 5 + + +@pytest.mark.parametrize( + "message_cls, expected_type", + [ + param(AccuracyRecordsMessage, MessageType.ACCURACY_RECORD, id="records"), + param( + ProcessAccuracyResultMessage, + MessageType.PROCESS_ACCURACY_RESULT, + id="result", + ), + ], +) # fmt: skip +def test_accuracy_message_types(message_cls, expected_type) -> None: + assert message_cls.model_fields["message_type"].default == expected_type + + +def test_accuracy_records_message_serializes_records() -> None: + message = AccuracyRecordsMessage( + service_id="records-manager", + records=[_make_record(session_num=1), _make_record(session_num=2)], + ) + assert message.message_type == MessageType.ACCURACY_RECORD + dumped = message.model_dump() + assert len(dumped["records"]) == 2 + assert dumped["records"][0]["session_num"] == 1 + + +def test_process_accuracy_result_message_carries_summary() -> None: + message = ProcessAccuracyResultMessage( + service_id="records-manager", + accuracy_result=ProcessAccuracyResult(results=_summary()), + ) + assert message.message_type == MessageType.PROCESS_ACCURACY_RESULT + assert message.accuracy_result.results.total_evaluated == 5 From bc2360d3f773922d80f2f36fdcb02c7bc44f7de5 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 20:04:53 -0700 Subject: [PATCH 12/39] feat(accuracy): add AccuracyAccumulator + AccuracyJSONLWriter channel Add a dedicated "accuracy" record-type channel alongside the existing shoehorn: an AccuracyAccumulator that rolls graded AccuracyRecordsData into an AccuracySummary (overall + per-task pass/unparsed rates, phase-scoped) and an AccuracyJSONLWriter that streams per-record grades to accuracy_export.jsonl. Registers both in plugins.yaml (accuracy accumulator, accuracy_jsonl_writer stream_exporter) and regenerates plugin enums. No producer feeds the new channel yet; the legacy accuracy_results path is untouched and stays green. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/accuracy/accumulator.py | 135 ++++++++++++++++ src/aiperf/accuracy/jsonl_writer.py | 61 ++++++++ src/aiperf/config/artifacts.py | 7 + src/aiperf/plugin/enums.py | 4 +- src/aiperf/plugin/plugins.yaml | 18 +++ tests/unit/accuracy/test_accumulator.py | 189 +++++++++++++++++++++++ tests/unit/accuracy/test_jsonl_writer.py | 70 +++++++++ 7 files changed, 482 insertions(+), 2 deletions(-) create mode 100644 src/aiperf/accuracy/accumulator.py create mode 100644 src/aiperf/accuracy/jsonl_writer.py create mode 100644 tests/unit/accuracy/test_accumulator.py create mode 100644 tests/unit/accuracy/test_jsonl_writer.py diff --git a/src/aiperf/accuracy/accumulator.py b/src/aiperf/accuracy/accumulator.py new file mode 100644 index 0000000000..777f1f91fd --- /dev/null +++ b/src/aiperf/accuracy/accumulator.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import bisect +from collections import defaultdict +from typing import TYPE_CHECKING, Any + +from aiperf.accuracy.models import ( + AccuracyRecordsData, + AccuracySummary, + TaskAccuracyStats, +) +from aiperf.common.exceptions import PostProcessorDisabled +from aiperf.post_processors.base_metrics_processor import BaseMetricsProcessor + +if TYPE_CHECKING: + from aiperf.common.accumulator_protocols import ExportContext, SummaryContext + from aiperf.common.enums import CreditPhase + from aiperf.config.resolution.plan import BenchmarkRun + + +class AccuracyAccumulator(BaseMetricsProcessor): + """Accumulate graded accuracy records on the dedicated ``accuracy`` channel. + + Ingests per-graded-response ``AccuracyRecordsData`` and rolls them up into an + ``AccuracySummary`` (overall + per-task pass rates and unparsed counts). + Because each record carries its own ``task`` label, this accumulator needs no + ``on_dataset_configured`` hook — task bucketing is read straight off the + record. + + Phase scoping mirrors ``ServerMetricsAccumulator``: ``export_results(ctx)`` + filters records to ``ctx.phase`` so warmup grades never leak into the + profiling summary (and vice versa). + + Raises: + PostProcessorDisabled: When accuracy mode is not enabled. + """ + + # RecordsManager routes phase-scoped-export accumulators through + # export_results(ctx) instead of the unscoped summarize(). + supports_phase_scoped_export = True + + def __init__(self, run: BenchmarkRun, **kwargs: Any) -> None: + if run.cfg.accuracy is None or not run.cfg.accuracy.enabled: + raise PostProcessorDisabled( + "Accuracy accumulator is disabled: accuracy mode is not enabled" + ) + super().__init__(run=run, **kwargs) + self.run = run + self._records: list[AccuracyRecordsData] = [] + self._timestamps_ns: list[int] = [] + + async def process_record(self, record: AccuracyRecordsData) -> None: + """Append a graded record, preserving per-worker insertion order.""" + self._records.append(record) + self._timestamps_ns.append(record.timestamp_ns) + + def query_time_range(self, start_ns: int, end_ns: int) -> list[AccuracyRecordsData]: + """Return records whose ``timestamp_ns`` is in ``[start_ns, end_ns)``. + + Analyzer query surface. Uses ``bisect`` on the append-ordered + ``_timestamps_ns`` (records arrive time-ordered per worker). + """ + lo = bisect.bisect_left(self._timestamps_ns, start_ns) + hi = bisect.bisect_left(self._timestamps_ns, end_ns) + return self._records[lo:hi] + + def _build_summary(self, phase: CreditPhase | None) -> AccuracySummary | None: + """Roll scoped records into an ``AccuracySummary`` (None when empty). + + ``phase`` selects one phase; ``None`` is phase-agnostic (all records). + Records with ``task is None`` count toward the overall totals but are + absent from ``per_task``. + """ + scoped = ( + self._records + if phase is None + else [r for r in self._records if r.benchmark_phase == phase] + ) + if not scoped: + return None + + total_evaluated = len(scoped) + total_passed = sum(1 for r in scoped if r.passed) + overall_unparsed = sum(1 for r in scoped if r.unparsed) + accuracy_rate = total_passed / total_evaluated if total_evaluated else 0.0 + + task_total: dict[str, int] = defaultdict(int) + task_passed: dict[str, int] = defaultdict(int) + task_unparsed: dict[str, int] = defaultdict(int) + for r in scoped: + if r.task is None: + continue + task_total[r.task] += 1 + if r.passed: + task_passed[r.task] += 1 + if r.unparsed: + task_unparsed[r.task] += 1 + + per_task: dict[str, TaskAccuracyStats] = {} + for task, total in task_total.items(): + passed = task_passed.get(task, 0) + unparsed = task_unparsed.get(task, 0) + per_task[task] = TaskAccuracyStats( + total=total, + passed=passed, + unparsed=unparsed, + accuracy_rate=passed / total if total else 0.0, + unparsed_rate=unparsed / total if total else 0.0, + ) + + return AccuracySummary( + total_evaluated=total_evaluated, + total_passed=total_passed, + accuracy_rate=accuracy_rate, + overall_unparsed=overall_unparsed, + grader_name=scoped[0].grader_name, + per_task=per_task, + ) + + async def export_results(self, ctx: ExportContext) -> AccuracySummary | None: + """Return an ``AccuracySummary`` scoped to ``ctx.phase`` (all if None). + + Returns None when no records fall in scope so RecordsManager can skip + publishing, mirroring ``ServerMetricsAccumulator.export_results``. + """ + return self._build_summary(ctx.phase) + + async def summarize( + self, ctx: SummaryContext | None = None + ) -> AccuracySummary | None: + """Return the phase-agnostic (all-phase) accuracy summary.""" + return self._build_summary(phase=None) diff --git a/src/aiperf/accuracy/jsonl_writer.py b/src/aiperf/accuracy/jsonl_writer.py new file mode 100644 index 0000000000..d00419225c --- /dev/null +++ b/src/aiperf/accuracy/jsonl_writer.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from aiperf.accuracy.models import AccuracyRecordsData +from aiperf.common.environment import Environment +from aiperf.common.exceptions import PostProcessorDisabled +from aiperf.common.mixins import BufferedJSONLWriterMixin +from aiperf.common.models import MetricResult +from aiperf.post_processors.base_metrics_processor import BaseMetricsProcessor + +if TYPE_CHECKING: + from aiperf.config.resolution.plan import BenchmarkRun + + +class AccuracyJSONLWriter( + BaseMetricsProcessor, BufferedJSONLWriterMixin[AccuracyRecordsData] +): + """Exports per-record graded accuracy data to JSONL files. + + Streams each ``AccuracyRecordsData`` as it arrives on the dedicated accuracy + channel, writing one JSON line per graded response. Each line carries the + grade (pass/unparsed/confidence), the expected/actual answers, and the + grader's reasoning, enabling per-response post-hoc analysis. + """ + + def __init__( + self, + run: BenchmarkRun, + **kwargs, + ) -> None: + if run.cfg.accuracy is None or not run.cfg.accuracy.enabled: + raise PostProcessorDisabled( + "Accuracy JSONL export is disabled: accuracy mode is not enabled" + ) + + output_file = run.cfg.artifacts.accuracy_export_jsonl_file + + super().__init__( + run=run, + output_file=output_file, + batch_size=Environment.RECORD.EXPORT_BATCH_SIZE, + **kwargs, + ) + + self.info(f"Accuracy JSONL export enabled: {self.output_file}") + + async def process_record(self, record: AccuracyRecordsData) -> None: + """Write a single graded accuracy record to the JSONL buffer.""" + await self.buffered_write(record) + + async def finalize(self) -> None: + """Flush any buffered data at end-of-run (``StreamExporterProtocol``).""" + await self.flush_buffer() + + async def summarize(self) -> list[MetricResult]: + """Summarize the results. This writer streams records, so returns [].""" + return [] diff --git a/src/aiperf/config/artifacts.py b/src/aiperf/config/artifacts.py index 27462e4994..c9237258e3 100644 --- a/src/aiperf/config/artifacts.py +++ b/src/aiperf/config/artifacts.py @@ -337,6 +337,13 @@ def server_metrics_export_jsonl_file(self) -> Path: name = f"{base}_server_metrics.jsonl" if base else "server_metrics_export.jsonl" return self.dir / name + @property + def accuracy_export_jsonl_file(self) -> Path: + """Path for the per-record accuracy JSONL export file.""" + base = self._base() + name = f"{base}_accuracy.jsonl" if base else "accuracy_export.jsonl" + return self.dir / name + @property def network_latency_export_jsonl_file(self) -> Path: """Path for the per-sample network latency RTT probe JSONL export file.""" diff --git a/src/aiperf/plugin/enums.py b/src/aiperf/plugin/enums.py index 7dce3a54b0..34288af258 100644 --- a/src/aiperf/plugin/enums.py +++ b/src/aiperf/plugin/enums.py @@ -79,11 +79,11 @@ AccumulatorTypeStr: TypeAlias = str AccumulatorType = plugins.create_enum(PluginType.ACCUMULATOR, "AccumulatorType", module=__name__) -"""Dynamic enum for accumulator. Example: AccumulatorType.ACCURACY_RESULTS, AccumulatorType.METRIC_RESULTS, AccumulatorType.SERVER_METRICS""" +"""Dynamic enum for accumulator. Example: AccumulatorType.ACCURACY, AccumulatorType.GPU_TELEMETRY, AccumulatorType.SERVER_METRICS""" StreamExporterTypeStr: TypeAlias = str StreamExporterType = plugins.create_enum(PluginType.STREAM_EXPORTER, "StreamExporterType", module=__name__) -"""Dynamic enum for stream exporter. Example: StreamExporterType.GPU_TELEMETRY_JSONL_WRITER, StreamExporterType.OTEL_METRICS_STREAMER, StreamExporterType.SERVER_METRICS_JSONL_WRITER""" +"""Dynamic enum for stream exporter. Example: StreamExporterType.ACCURACY_JSONL_WRITER, StreamExporterType.OTEL_METRICS_STREAMER, StreamExporterType.SERVER_METRICS_JSONL_WRITER""" AnalyzerTypeStr: TypeAlias = str AnalyzerType = plugins.create_enum(PluginType.ANALYZER, "AnalyzerType", module=__name__) diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index 21f60c4fda..51eda4b82b 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -1028,6 +1028,15 @@ accumulator: metadata: record_types: [metric_records] + accuracy: + class: aiperf.accuracy.accumulator:AccuracyAccumulator + description: | + Accuracy accumulator that aggregates graded accuracy records on the + dedicated accuracy channel and computes overall/per-task pass rates and + unparsed counts. Loaded when accuracy evaluation is enabled. + metadata: + record_types: [accuracy] + # ============================================================================= # Stream Exporters # ============================================================================= @@ -1045,6 +1054,15 @@ stream_exporter: metadata: record_types: [gpu_telemetry] + accuracy_jsonl_writer: + class: aiperf.accuracy.jsonl_writer:AccuracyJSONLWriter + description: | + Accuracy JSONL writer that exports per-record graded accuracy data + (pass/unparsed/confidence/expected/actual/reasoning) to a JSONL file. + Enabled when accuracy evaluation is enabled. + metadata: + record_types: [accuracy] + network_latency_jsonl_writer: class: aiperf.network_latency.jsonl_writer:NetworkLatencyJSONLWriter description: | diff --git a/tests/unit/accuracy/test_accumulator.py b/tests/unit/accuracy/test_accumulator.py new file mode 100644 index 0000000000..dde032435e --- /dev/null +++ b/tests/unit/accuracy/test_accumulator.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import pytest + +from aiperf.accuracy.accumulator import AccuracyAccumulator +from aiperf.accuracy.models import AccuracyRecordsData, AccuracySummary +from aiperf.common.accumulator_protocols import ExportContext +from aiperf.common.enums import CreditPhase +from aiperf.common.exceptions import PostProcessorDisabled +from aiperf.plugin.enums import AccuracyBenchmarkType, EndpointType +from tests.unit.conftest import make_benchmark_run + + +def _make_accumulator() -> AccuracyAccumulator: + return AccuracyAccumulator( + run=make_benchmark_run( + model_names=["test-model"], + endpoint_type=EndpointType.COMPLETIONS, + streaming=False, + accuracy={"benchmark": AccuracyBenchmarkType.MMLU}, + ) + ) + + +def _record( + *, + timestamp_ns: int, + phase: CreditPhase = CreditPhase.PROFILING, + task: str | None = "math", + passed: bool = True, + unparsed: bool = False, + session_num: int = 0, + grader_name: str = "exact_match", +) -> AccuracyRecordsData: + return AccuracyRecordsData( + session_num=session_num, + worker_id="worker-0", + benchmark_phase=phase, + timestamp_ns=timestamp_ns, + task=task, + grader_name=grader_name, + passed=passed, + unparsed=unparsed, + confidence=1.0, + expected="A", + actual="A", + reasoning="matched", + ) + + +async def _seed(acc: AccuracyAccumulator, records: list[AccuracyRecordsData]) -> None: + for record in records: + await acc.process_record(record) + + +@pytest.mark.asyncio +class TestAccuracyAccumulator: + async def test_disabled_raises(self) -> None: + with pytest.raises(PostProcessorDisabled): + AccuracyAccumulator( + run=make_benchmark_run( + model_names=["test-model"], + endpoint_type=EndpointType.COMPLETIONS, + streaming=False, + ) + ) + + async def test_export_results_profiling_overall_and_per_task(self) -> None: + acc = _make_accumulator() + await _seed( + acc, + [ + _record(timestamp_ns=10, task="math", passed=True), + _record(timestamp_ns=20, task="math", passed=False, unparsed=True), + _record(timestamp_ns=30, task="algebra", passed=True), + _record(timestamp_ns=40, task="algebra", passed=True, unparsed=True), + # WARMUP record must be excluded from PROFILING scope. + _record( + timestamp_ns=5, + phase=CreditPhase.WARMUP, + task="math", + passed=False, + ), + ], + ) + + summary = await acc.export_results(ExportContext(phase=CreditPhase.PROFILING)) + + assert isinstance(summary, AccuracySummary) + assert summary.total_evaluated == 4 + assert summary.total_passed == 3 + assert summary.overall_unparsed == 2 + assert summary.accuracy_rate == pytest.approx(0.75) + assert summary.grader_name == "exact_match" + + assert set(summary.per_task) == {"math", "algebra"} + + math = summary.per_task["math"] + assert math.total == 2 + assert math.passed == 1 + assert math.unparsed == 1 + assert math.accuracy_rate == pytest.approx(0.5) + assert math.unparsed_rate == pytest.approx(0.5) + + algebra = summary.per_task["algebra"] + assert algebra.total == 2 + assert algebra.passed == 2 + assert algebra.unparsed == 1 + assert algebra.accuracy_rate == pytest.approx(1.0) + assert algebra.unparsed_rate == pytest.approx(0.5) + + async def test_warmup_scope_isolated(self) -> None: + acc = _make_accumulator() + await _seed( + acc, + [ + _record(timestamp_ns=5, phase=CreditPhase.WARMUP, passed=True), + _record(timestamp_ns=15, phase=CreditPhase.PROFILING, passed=False), + ], + ) + + warmup = await acc.export_results(ExportContext(phase=CreditPhase.WARMUP)) + assert warmup is not None + assert warmup.total_evaluated == 1 + assert warmup.total_passed == 1 + assert warmup.accuracy_rate == pytest.approx(1.0) + + async def test_phase_with_no_records_returns_none(self) -> None: + acc = _make_accumulator() + await _seed( + acc, + [_record(timestamp_ns=15, phase=CreditPhase.PROFILING)], + ) + + assert await acc.export_results(ExportContext(phase=CreditPhase.WARMUP)) is None + + async def test_empty_returns_none(self) -> None: + acc = _make_accumulator() + assert await acc.export_results(ExportContext(phase=None)) is None + + async def test_task_none_counts_overall_but_absent_from_per_task(self) -> None: + acc = _make_accumulator() + await _seed( + acc, + [ + _record(timestamp_ns=10, task=None, passed=True), + _record(timestamp_ns=20, task="math", passed=False), + ], + ) + + summary = await acc.export_results(ExportContext(phase=CreditPhase.PROFILING)) + assert summary is not None + assert summary.total_evaluated == 2 + assert summary.total_passed == 1 + assert set(summary.per_task) == {"math"} + + async def test_summarize_is_phase_agnostic(self) -> None: + acc = _make_accumulator() + await _seed( + acc, + [ + _record(timestamp_ns=5, phase=CreditPhase.WARMUP, passed=True), + _record(timestamp_ns=15, phase=CreditPhase.PROFILING, passed=False), + ], + ) + + summary = await acc.summarize() + assert summary is not None + assert summary.total_evaluated == 2 + assert summary.total_passed == 1 + assert summary.accuracy_rate == pytest.approx(0.5) + + async def test_query_time_range_slices_by_timestamp(self) -> None: + acc = _make_accumulator() + await _seed( + acc, + [ + _record(timestamp_ns=10), + _record(timestamp_ns=20), + _record(timestamp_ns=30), + _record(timestamp_ns=40), + ], + ) + + sliced = acc.query_time_range(20, 40) + assert [r.timestamp_ns for r in sliced] == [20, 30] diff --git a/tests/unit/accuracy/test_jsonl_writer.py b/tests/unit/accuracy/test_jsonl_writer.py new file mode 100644 index 0000000000..00801a8bd4 --- /dev/null +++ b/tests/unit/accuracy/test_jsonl_writer.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import orjson +import pytest + +from aiperf.accuracy.jsonl_writer import AccuracyJSONLWriter +from aiperf.accuracy.models import AccuracyRecordsData +from aiperf.common.enums import CreditPhase +from aiperf.common.exceptions import PostProcessorDisabled +from aiperf.plugin.enums import AccuracyBenchmarkType, EndpointType +from tests.unit.conftest import make_benchmark_run + + +def _record(*, timestamp_ns: int, task: str | None = "math") -> AccuracyRecordsData: + return AccuracyRecordsData( + session_num=0, + worker_id="worker-0", + benchmark_phase=CreditPhase.PROFILING, + timestamp_ns=timestamp_ns, + task=task, + grader_name="exact_match", + passed=True, + unparsed=False, + confidence=0.42, + expected="A", + actual="A", + reasoning="the answer is A", + ) + + +@pytest.mark.asyncio +class TestAccuracyJSONLWriter: + async def test_disabled_raises(self) -> None: + with pytest.raises(PostProcessorDisabled): + AccuracyJSONLWriter( + run=make_benchmark_run( + model_names=["test-model"], + endpoint_type=EndpointType.COMPLETIONS, + streaming=False, + ) + ) + + async def test_records_round_trip(self) -> None: + run = make_benchmark_run( + model_names=["test-model"], + endpoint_type=EndpointType.COMPLETIONS, + streaming=False, + accuracy={"benchmark": AccuracyBenchmarkType.MMLU}, + ) + writer = AccuracyJSONLWriter(run=run) + await writer.initialize() + await writer.start() + + records = [_record(timestamp_ns=10), _record(timestamp_ns=20, task="algebra")] + for record in records: + await writer.process_record(record) + await writer.finalize() + await writer.stop() + + lines = writer.output_file.read_text().splitlines() + assert len(lines) == 2 + + parsed = [orjson.loads(line) for line in lines] + assert parsed[0]["timestamp_ns"] == 10 + assert parsed[0]["reasoning"] == "the answer is A" + assert parsed[0]["confidence"] == pytest.approx(0.42) + assert parsed[1]["task"] == "algebra" From a3a86b8788d078d0ca256407d151a41b085f723a Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 20:16:49 -0700 Subject: [PATCH 13/39] feat(accuracy): light up dedicated accuracy channel end-to-end Producer emits typed AccuracyRecordsData; RecordProcessorService partitions dict vs typed records generically and pushes a dedicated AccuracyRecordsMessage; RecordsManager ingests via routing and publishes ProcessAccuracyResultMessage. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- .../accuracy/accuracy_record_processor.py | 42 ++++-- .../records/record_processor_service.py | 53 +++++++- src/aiperf/records/records_manager.py | 43 ++++++ .../test_accuracy_record_processor.py | 50 +++++-- .../records/test_record_processor_service.py | 101 +++++++++++++- .../records/test_records_manager_accuracy.py | 128 ++++++++++++++++++ 6 files changed, 383 insertions(+), 34 deletions(-) create mode 100644 tests/unit/records/test_records_manager_accuracy.py diff --git a/src/aiperf/accuracy/accuracy_record_processor.py b/src/aiperf/accuracy/accuracy_record_processor.py index 38c9f73f1e..1884fb7b19 100644 --- a/src/aiperf/accuracy/accuracy_record_processor.py +++ b/src/aiperf/accuracy/accuracy_record_processor.py @@ -6,14 +6,12 @@ from typing import TYPE_CHECKING, Any from aiperf.accuracy.models import ( - ACCURACY_RECORD_CORRECT_KEY, - ACCURACY_RECORD_UNPARSED_KEY, + AccuracyRecordsData, GradingResult, ) from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.mixins import AIPerfLifecycleMixin from aiperf.common.models import MetricRecordMetadata, ParsedResponseRecord -from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.plugin import plugins from aiperf.plugin.enums import PluginType @@ -58,8 +56,10 @@ def __init__( grader_cls = plugins.get_class(PluginType.ACCURACY_GRADER, grader_name) self.grader: BaseGrader = grader_cls(run=run) + self._grader_name = grader_name self._verbose = acc_cfg.verbose self._ground_truths: list[str] | None = None + self._tasks: list[str] | None = None def on_dataset_configured(self, metadata: DatasetMetadata) -> None: """Receive ground-truth answers from the DatasetConfiguredNotification. @@ -73,17 +73,20 @@ def on_dataset_configured(self, metadata: DatasetMetadata) -> None: for c in metadata.conversations if c.accuracy_ground_truth is not None ] + self._tasks = [ + c.accuracy_task + for c in metadata.conversations + if c.accuracy_task is not None + ] async def process_record( self, record: ParsedResponseRecord, metadata: MetricRecordMetadata - ) -> MetricRecordDict: + ) -> AccuracyRecordsData: """Grade a single response against its corresponding benchmark problem. Maps ``metadata.session_num % len(_ground_truths)`` to the ground-truth - answer, runs the configured grader, and returns a MetricRecordDict - containing the per-record transport keys ``accuracy_correct`` and - ``accuracy_unparsed`` (underscore-namespaced so they never collide with - the ``accuracy.`` summary tags). + answer, runs the configured grader, and returns a typed + ``AccuracyRecordsData`` that flows on the dedicated ``accuracy`` channel. Raises: RuntimeError: if on_dataset_configured was not called before processing. @@ -93,7 +96,6 @@ async def process_record( "AccuracyRecordProcessor: dataset not configured; " "on_dataset_configured must be called before process_record" ) - record_metrics = MetricRecordDict() ground_truth = self._ground_truths[ metadata.session_num % len(self._ground_truths) @@ -102,12 +104,28 @@ async def process_record( result: GradingResult = await self.grader.grade(response_text, ground_truth) - record_metrics[ACCURACY_RECORD_CORRECT_KEY] = 1.0 if result.correct else 0.0 - record_metrics[ACCURACY_RECORD_UNPARSED_KEY] = 1.0 if result.unparsed else 0.0 + task = ( + self._tasks[metadata.session_num % len(self._tasks)] + if self._tasks + else None + ) self._log_grading_detail(metadata.session_num, response_text, result) - return record_metrics + return AccuracyRecordsData( + session_num=metadata.session_num, + worker_id=metadata.worker_id, + benchmark_phase=metadata.benchmark_phase, + timestamp_ns=metadata.request_end_ns, + task=task, + grader_name=self._grader_name, + passed=result.correct, + unparsed=result.unparsed, + confidence=result.confidence, + expected=result.ground_truth, + actual=result.extracted_answer, + reasoning=result.reasoning, + ) def _log_grading_detail( self, session_num: int, response_text: str, result: GradingResult diff --git a/src/aiperf/records/record_processor_service.py b/src/aiperf/records/record_processor_service.py index 305a42f7d1..4e1dbedeed 100644 --- a/src/aiperf/records/record_processor_service.py +++ b/src/aiperf/records/record_processor_service.py @@ -2,8 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio import traceback -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any +from aiperf.accuracy.models import AccuracyRecordsData from aiperf.common.base_component_service import BaseComponentService from aiperf.common.enums import ( CommAddress, @@ -16,6 +17,7 @@ from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.hooks import on_command, on_message, on_pull_message from aiperf.common.messages import ( + AccuracyRecordsMessage, DatasetConfiguredNotification, InferenceResultsMessage, MetricRecordsMessage, @@ -286,16 +288,55 @@ async def _process_and_forward_record( else: results.append(result) + # Partition processor outputs by transport: MetricRecordDict is a dict + # subclass and merges into MetricRecordsMessage.results; typed Pydantic + # record models (e.g. AccuracyRecordsData) travel on dedicated channels. + # A typed record leaking into results would break to_data()'s dict-merge. + metric_dicts = [r for r in results if isinstance(r, dict)] + typed_records = [r for r in results if not isinstance(r, dict)] + await self.records_push_client.push( MetricRecordsMessage( service_id=self.service_id, metadata=metadata, - results=results, + results=metric_dicts, trace_data=trace_data, error=error, ) ) + await self._push_typed_records(typed_records) + + async def _push_typed_records( + self, typed_records: list[AccuracyRecordsData] + ) -> None: + """Push typed record models on their dedicated channels. + + Groups records by their ``record_type`` classvar and builds one dedicated + message per type via a generic ``record_type -> message builder`` mapping, + so future typed records reuse this seam instead of adding an if-branch. + """ + if not typed_records: + return + + builders = { + AccuracyRecordsData.record_type: lambda records: AccuracyRecordsMessage( + service_id=self.service_id, + records=records, + ), + } + + grouped: dict[str, list[Any]] = {} + for record in typed_records: + grouped.setdefault(record.record_type, []).append(record) + + for record_type, records in grouped.items(): + builder = builders.get(record_type) + if builder is None: + self.error(f"No message builder for typed record type: {record_type}") + continue + await self.records_push_client.push(builder(records)) + async def _forward_failed_record( self, message: InferenceResultsMessage, @@ -369,15 +410,15 @@ def _free_record_data( async def _process_record( self, record: ParsedResponseRecord, metadata: MetricRecordMetadata - ) -> list[MetricRecordDict | BaseException]: + ) -> list[MetricRecordDict | AccuracyRecordsData | BaseException]: """Stream a record to the records processors.""" tasks = [ processor.process_record(record, metadata) for processor in self.records_processors ] - results: list[MetricRecordDict | BaseException | None] = await asyncio.gather( - *tasks, return_exceptions=True - ) + results: list[ + MetricRecordDict | AccuracyRecordsData | BaseException | None + ] = await asyncio.gather(*tasks, return_exceptions=True) return [result for result in results if result is not None] diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 419b59dd49..427eec10e6 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -8,6 +8,7 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any +from aiperf.accuracy.models import ProcessAccuracyResult from aiperf.common.accumulator_protocols import ( AccumulatorProtocol, ExportContext, @@ -25,11 +26,13 @@ from aiperf.common.environment import Environment from aiperf.common.hooks import background_task, on_command, on_message, on_pull_message from aiperf.common.messages import ( + AccuracyRecordsMessage, AllRecordsReceivedMessage, DatasetConfiguredNotification, MetricRecordsData, MetricRecordsMessage, NetworkLatencyRecordMessage, + ProcessAccuracyResultMessage, ProcessAllResultsMessage, ProcessRecordsCommand, ProcessRecordsResultMessage, @@ -458,6 +461,7 @@ def __init__( self._server_metrics_accumulator = self._accumulators.get( AccumulatorType.SERVER_METRICS ) + self._accuracy_accumulator = self._accumulators.get(AccumulatorType.ACCURACY) def _build_routing_table(self) -> dict[str, list[Any]]: """Build record_type string -> handler mapping from plugin metadata.""" @@ -585,6 +589,18 @@ async def _on_server_metrics_records( elif message.error: self._server_metrics_state.error_counts[message.error] += 1 + @on_pull_message(MessageType.ACCURACY_RECORD) + async def _on_accuracy_records(self, message: AccuracyRecordsMessage) -> None: + """Handle graded accuracy records from a record processor. + + The routing table auto-includes ``"accuracy"`` from plugin metadata, so + ``_dispatch_record`` fans each record out to the AccuracyAccumulator and + AccuracyJSONLWriter without any accuracy-specific routing code here. + """ + for record in message.records: + for error in await self._dispatch_record(record): + self.debug(lambda e=error: f"Accuracy record dispatch error: {e!r}") + @on_pull_message(MessageType.NETWORK_LATENCY_RECORD) async def _on_network_latency_records( self, message: NetworkLatencyRecordMessage @@ -1409,6 +1425,14 @@ async def _process_results( except Exception as e: self.exception(f"Failed to publish server metrics results: {e!r}") + if self.run.cfg.accuracy is None or not self.run.cfg.accuracy.enabled: + self.debug("Accuracy evaluation is disabled, skipping publish") + else: + try: + await self._publish_accuracy_results(phase) + except Exception as e: + self.exception(f"Failed to publish accuracy results: {e!r}") + # Publish the unified ProcessAllResultsMessage over the populated # accumulators. The per-stream result messages above remain the # shutdown trigger; this is supplementary. @@ -1468,6 +1492,25 @@ async def _publish_telemetry_results(self, phase: CreditPhase) -> None: ) ) + async def _publish_accuracy_results(self, phase: CreditPhase) -> None: + """Publish phase-scoped accuracy results on the dedicated accuracy channel. + + Mirrors ``_publish_telemetry_results``: exports the phase-scoped accuracy + summary from the AccuracyAccumulator and publishes it independently from + inference results. + """ + if self._accuracy_accumulator is None: + return + summary = await self._accuracy_accumulator.export_results( + ExportContext(phase=phase) + ) + await self.publish( + ProcessAccuracyResultMessage( + service_id=self.service_id, + accuracy_result=ProcessAccuracyResult(results=summary), + ) + ) + async def _process_server_metrics_results(self) -> ProcessServerMetricsResult: """Process server metrics results by exporting the accumulated server metrics data. diff --git a/tests/unit/accuracy/test_accuracy_record_processor.py b/tests/unit/accuracy/test_accuracy_record_processor.py index 054327228b..3bfd901bf3 100644 --- a/tests/unit/accuracy/test_accuracy_record_processor.py +++ b/tests/unit/accuracy/test_accuracy_record_processor.py @@ -7,7 +7,7 @@ from aiperf.accuracy.accuracy_record_processor import AccuracyRecordProcessor from aiperf.accuracy.accuracy_results_processor import AccuracyResultsProcessor -from aiperf.accuracy.models import GradingResult +from aiperf.accuracy.models import AccuracyRecordsData, GradingResult from aiperf.common.enums import CreditPhase from aiperf.common.messages.inference_messages import MetricRecordsData from aiperf.common.models.dataset_models import ConversationMetadata, DatasetMetadata @@ -85,6 +85,7 @@ def test_populates_ground_truths_from_metadata(self, monkeypatch) -> None: processor.on_dataset_configured(metadata) assert processor._ground_truths == ["A", "B", "C"] + assert processor._tasks == ["t1", "t2", "t3"] def test_skips_conversations_without_accuracy_fields(self, monkeypatch) -> None: processor = _make_processor(monkeypatch) @@ -128,40 +129,60 @@ async def test_process_record_wraps_when_session_num_exceeds_dataset( metadata = create_metric_metadata(session_num=1) result = await processor.process_record(sample_parsed_record, metadata) - assert result["accuracy_correct"] == 1.0 - assert result["accuracy_unparsed"] == 0.0 + assert isinstance(result, AccuracyRecordsData) + assert result.passed is True + assert result.unparsed is False processor.grader.grade.assert_awaited_once_with("Hello world", "A") - async def test_process_record_wraps_to_correct_problem( + async def test_process_record_maps_all_grading_and_metadata_fields( self, monkeypatch, sample_parsed_record ) -> None: - """With N problems, session_num=N+1 grades problem at index 1.""" + """Every AccuracyRecordsData field maps from the grader result / metadata.""" processor = _make_processor(monkeypatch) processor._ground_truths = ["A", "B", "C"] + processor._tasks = ["t0", "t1", "t2"] grading_result = GradingResult( correct=False, unparsed=True, - confidence=1.0, - reasoning="Wrong", + confidence=0.42, + reasoning="Wrong answer", extracted_answer="A", ground_truth="B", ) processor.grader.grade = AsyncMock(return_value=grading_result) - # session_num=4 % 3 = index 1 (ground_truth="B") - metadata = create_metric_metadata(session_num=4) + # session_num=4 % 3 = index 1 -> ground_truth="B", task="t1" + metadata = create_metric_metadata( + session_num=4, + worker_id="worker-9", + request_end_ns=1_234_567_890, + benchmark_phase=CreditPhase.PROFILING, + ) result = await processor.process_record(sample_parsed_record, metadata) - assert result["accuracy_correct"] == 0.0 - assert result["accuracy_unparsed"] == 1.0 + assert isinstance(result, AccuracyRecordsData) + assert result.session_num == 4 + assert result.worker_id == "worker-9" + assert result.benchmark_phase == CreditPhase.PROFILING + assert result.timestamp_ns == 1_234_567_890 + assert result.task == "t1" + assert result.grader_name == "multiple_choice" + assert result.passed is False + assert result.unparsed is True + assert result.confidence == 0.42 + assert result.expected == "B" + assert result.actual == "A" + assert result.reasoning == "Wrong answer" processor.grader.grade.assert_awaited_once_with("Hello world", "B") - async def test_process_record_last_valid_session_num_succeeds( + async def test_process_record_task_none_when_no_tasks( self, monkeypatch, sample_parsed_record ) -> None: + """task is None when the dataset carried no task labels.""" processor = _make_processor(monkeypatch) processor._ground_truths = ["A", "B"] + processor._tasks = [] grading_result = GradingResult( correct=True, @@ -175,8 +196,9 @@ async def test_process_record_last_valid_session_num_succeeds( metadata = create_metric_metadata(session_num=1) result = await processor.process_record(sample_parsed_record, metadata) - assert result["accuracy_correct"] == 1.0 - assert result["accuracy_unparsed"] == 0.0 + assert isinstance(result, AccuracyRecordsData) + assert result.task is None + assert result.passed is True async def test_process_record_raises_if_not_configured( self, monkeypatch, sample_parsed_record diff --git a/tests/unit/records/test_record_processor_service.py b/tests/unit/records/test_record_processor_service.py index dea108cbe1..73c3b8f8c7 100644 --- a/tests/unit/records/test_record_processor_service.py +++ b/tests/unit/records/test_record_processor_service.py @@ -6,10 +6,107 @@ import pytest -from aiperf.common.enums import CreditPhase -from aiperf.common.messages import BaseServiceErrorMessage +from aiperf.accuracy.models import AccuracyRecordsData +from aiperf.common.enums import CreditPhase, ExportLevel +from aiperf.common.messages import ( + AccuracyRecordsMessage, + BaseServiceErrorMessage, + MetricRecordsMessage, +) from aiperf.common.utils import compute_time_ns +from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.records.record_processor_service import RecordProcessor +from tests.unit.post_processors.conftest import create_metric_metadata + + +def _make_accuracy_record(session_num: int = 0) -> AccuracyRecordsData: + return AccuracyRecordsData( + session_num=session_num, + worker_id="w1", + benchmark_phase=CreditPhase.PROFILING, + timestamp_ns=1_000, + task=None, + grader_name="multiple_choice", + passed=True, + unparsed=False, + confidence=1.0, + expected="A", + actual="A", + reasoning="ok", + ) + + +class TestRecordProcessorTypedRecordPartition: + """Processor outputs are partitioned by transport: MetricRecordDict (a dict + subclass) stays in MetricRecordsMessage.results; typed Pydantic records + (AccuracyRecordsData) travel on their own dedicated channel message.""" + + @pytest.mark.asyncio + async def test_process_and_forward_splits_dicts_from_typed_records(self): + """A mixed result list pushes the dict in MetricRecordsMessage.results and + the accuracy record in a separate AccuracyRecordsMessage.""" + metric_dict = MetricRecordDict({"some_metric": 1.0}) + accuracy_record = _make_accuracy_record() + + mock_self = MagicMock(spec=RecordProcessor) + mock_self.service_id = "rp" + mock_self.run = MagicMock() + mock_self.run.cfg.artifacts.export_level = ExportLevel.RECORDS + mock_self.records_push_client = AsyncMock() + mock_self.inference_result_parser = MagicMock() + mock_self.inference_result_parser.parse_request_record = AsyncMock( + return_value=MagicMock() + ) + mock_self._create_metric_record_metadata = MagicMock( + return_value=create_metric_metadata(session_num=0) + ) + mock_self._process_record = AsyncMock( + return_value=[metric_dict, accuracy_record] + ) + mock_self._free_record_data = MagicMock(return_value=(None, None)) + # Run the real partition/push seam against this mock instance. + mock_self._push_typed_records = ( + lambda recs: RecordProcessor._push_typed_records(mock_self, recs) + ) + + await RecordProcessor._process_and_forward_record( + mock_self, MagicMock(service_id="w1"), MagicMock(), None + ) + + pushed = [c.args[0] for c in mock_self.records_push_client.push.await_args_list] + metric_msgs = [m for m in pushed if isinstance(m, MetricRecordsMessage)] + accuracy_msgs = [m for m in pushed if isinstance(m, AccuracyRecordsMessage)] + + assert len(metric_msgs) == 1 + assert metric_msgs[0].results == [metric_dict] + assert all(isinstance(r, dict) for r in metric_msgs[0].results) + + assert len(accuracy_msgs) == 1 + assert accuracy_msgs[0].records == [accuracy_record] + + @pytest.mark.asyncio + async def test_push_typed_records_groups_by_record_type(self): + """All accuracy records land in a single AccuracyRecordsMessage.""" + mock_self = MagicMock(spec=RecordProcessor) + mock_self.service_id = "rp" + mock_self.records_push_client = AsyncMock() + + records = [_make_accuracy_record(0), _make_accuracy_record(1)] + await RecordProcessor._push_typed_records(mock_self, records) + + mock_self.records_push_client.push.assert_awaited_once() + msg = mock_self.records_push_client.push.await_args.args[0] + assert isinstance(msg, AccuracyRecordsMessage) + assert msg.records == records + + @pytest.mark.asyncio + async def test_push_typed_records_empty_is_noop(self): + mock_self = MagicMock(spec=RecordProcessor) + mock_self.records_push_client = AsyncMock() + + await RecordProcessor._push_typed_records(mock_self, []) + + mock_self.records_push_client.push.assert_not_awaited() class TestRecordProcessorCreateMetricRecordMetadata: diff --git a/tests/unit/records/test_records_manager_accuracy.py b/tests/unit/records/test_records_manager_accuracy.py new file mode 100644 index 0000000000..c4ee7c9427 --- /dev/null +++ b/tests/unit/records/test_records_manager_accuracy.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from aiperf.accuracy.models import AccuracyRecordsData, AccuracySummary +from aiperf.common.enums import CreditPhase +from aiperf.common.messages import ( + AccuracyRecordsMessage, + ProcessAccuracyResultMessage, +) +from aiperf.records.records_manager import RecordsManager + + +def _make_accuracy_record(session_num: int = 0) -> AccuracyRecordsData: + return AccuracyRecordsData( + session_num=session_num, + worker_id="w1", + benchmark_phase=CreditPhase.PROFILING, + timestamp_ns=1_000, + task=None, + grader_name="multiple_choice", + passed=True, + unparsed=False, + confidence=1.0, + expected="A", + actual="A", + reasoning="ok", + ) + + +class TestOnAccuracyRecords: + @pytest.mark.asyncio + async def test_dispatches_each_record(self) -> None: + mgr = MagicMock() + mgr.debug = MagicMock() + mgr._dispatch_record = AsyncMock(return_value=[]) + mgr._on_accuracy_records = RecordsManager._on_accuracy_records.__get__(mgr) + + records = [_make_accuracy_record(0), _make_accuracy_record(1)] + await mgr._on_accuracy_records( + AccuracyRecordsMessage(service_id="rp", records=records) + ) + + assert mgr._dispatch_record.await_count == 2 + dispatched = [c.args[0] for c in mgr._dispatch_record.await_args_list] + assert dispatched == records + + @pytest.mark.asyncio + async def test_dispatch_errors_are_logged_not_raised(self) -> None: + mgr = MagicMock() + mgr.debug = MagicMock() + mgr._dispatch_record = AsyncMock(return_value=[ValueError("boom")]) + mgr._on_accuracy_records = RecordsManager._on_accuracy_records.__get__(mgr) + + await mgr._on_accuracy_records( + AccuracyRecordsMessage(service_id="rp", records=[_make_accuracy_record()]) + ) + + assert mgr.debug.called + + +class TestPublishAccuracyResults: + @pytest.mark.asyncio + async def test_publishes_summary_from_accumulator(self) -> None: + summary = AccuracySummary( + total_evaluated=3, + total_passed=2, + accuracy_rate=2 / 3, + overall_unparsed=1, + grader_name="multiple_choice", + ) + accumulator = MagicMock() + accumulator.export_results = AsyncMock(return_value=summary) + + mgr = MagicMock() + mgr.service_id = "rm" + mgr.publish = AsyncMock() + mgr._accuracy_accumulator = accumulator + mgr._publish_accuracy_results = ( + RecordsManager._publish_accuracy_results.__get__(mgr) + ) + + await mgr._publish_accuracy_results(CreditPhase.PROFILING) + + accumulator.export_results.assert_awaited_once() + ctx = accumulator.export_results.await_args.args[0] + assert ctx.phase == CreditPhase.PROFILING + + mgr.publish.assert_awaited_once() + msg = mgr.publish.await_args.args[0] + assert isinstance(msg, ProcessAccuracyResultMessage) + assert msg.accuracy_result.results == summary + + @pytest.mark.asyncio + async def test_publishes_none_summary(self) -> None: + accumulator = MagicMock() + accumulator.export_results = AsyncMock(return_value=None) + + mgr = MagicMock() + mgr.service_id = "rm" + mgr.publish = AsyncMock() + mgr._accuracy_accumulator = accumulator + mgr._publish_accuracy_results = ( + RecordsManager._publish_accuracy_results.__get__(mgr) + ) + + await mgr._publish_accuracy_results(CreditPhase.PROFILING) + + mgr.publish.assert_awaited_once() + msg = mgr.publish.await_args.args[0] + assert msg.accuracy_result.results is None + + @pytest.mark.asyncio + async def test_no_accumulator_is_noop(self) -> None: + mgr = MagicMock() + mgr.publish = AsyncMock() + mgr._accuracy_accumulator = None + mgr._publish_accuracy_results = ( + RecordsManager._publish_accuracy_results.__get__(mgr) + ) + + await mgr._publish_accuracy_results(CreditPhase.PROFILING) + + mgr.publish.assert_not_awaited() From efaadac73463bb5b3bbffc74306921f73c901022 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 20:28:08 -0700 Subject: [PATCH 14/39] feat(accuracy): deliver AccuracySummary to exporters via SystemController Publish exactly one ProcessAccuracyResultMessage (profiling phase only), thread the AccuracySummary through SystemController -> ExporterConfig -> ExporterManager with shutdown wait-gating mirroring telemetry/server metrics, and rewrite the accuracy data/console exporters to render from the structured AccuracySummary instead of filtering accuracy.-prefixed MetricResults. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- .../accuracy/accuracy_console_exporter.py | 88 ++++------- src/aiperf/accuracy/accuracy_data_exporter.py | 72 ++------- src/aiperf/controller/system_controller.py | 35 ++++- src/aiperf/exporters/exporter_config.py | 2 + src/aiperf/exporters/exporter_manager.py | 3 + src/aiperf/records/records_manager.py | 9 +- .../test_accuracy_console_exporter.py | 140 +++++++----------- .../accuracy/test_accuracy_data_exporter.py | 114 ++++++-------- 8 files changed, 188 insertions(+), 275 deletions(-) diff --git a/src/aiperf/accuracy/accuracy_console_exporter.py b/src/aiperf/accuracy/accuracy_console_exporter.py index 1a5458ead5..435f55aee6 100644 --- a/src/aiperf/accuracy/accuracy_console_exporter.py +++ b/src/aiperf/accuracy/accuracy_console_exporter.py @@ -5,13 +5,6 @@ from typing import TYPE_CHECKING, Any -from aiperf.accuracy.models import ( - ACCURACY_METRIC_PREFIX, - ACCURACY_OVERALL_TAG, - ACCURACY_TASK_TAG_PREFIX, - ACCURACY_UNPARSED_TAG, - ACCURACY_UNPARSED_TASK_TAG_PREFIX, -) from aiperf.common.exceptions import ConsoleExporterDisabled from aiperf.common.mixins import AIPerfLoggerMixin from aiperf.exporters.exporter_config import ExporterConfig @@ -19,11 +12,15 @@ if TYPE_CHECKING: from rich.console import Console + from aiperf.accuracy.models import AccuracySummary + class AccuracyConsoleExporter(AIPerfLoggerMixin): """Console exporter for accuracy benchmarking results. - Renders a Rich table with per-task accuracy breakdown and overall score. + Renders a Rich table with per-task accuracy breakdown and overall score, + sourced from the structured ``AccuracySummary`` delivered on the dedicated + accuracy channel. """ def __init__(self, exporter_config: ExporterConfig, **kwargs: Any) -> None: @@ -39,37 +36,15 @@ def __init__(self, exporter_config: ExporterConfig, **kwargs: Any) -> None: async def export(self, console: Console) -> None: """Render accuracy results as a Rich table to the given console. - Prints a per-task breakdown (correct / total / accuracy%) followed by an - OVERALL row. Does nothing if no ``accuracy.*`` metrics are present in - ``exporter_config.results``. + Prints a per-task breakdown (passed / total / accuracy%) followed by an + OVERALL row. Does nothing when no accuracy summary was delivered. """ from rich.table import Table - results = self.exporter_config.results - if results is None or results.records is None: - return - - accuracy_metrics = [ - r for r in results.records if r.tag.startswith(ACCURACY_METRIC_PREFIX) - ] - if not accuracy_metrics: + summary = self.exporter_config.accuracy_results + if summary is None: return - overall = next( - (m for m in accuracy_metrics if m.tag == ACCURACY_OVERALL_TAG), None - ) - task_metrics = [ - m for m in accuracy_metrics if m.tag.startswith(ACCURACY_TASK_TAG_PREFIX) - ] - unparsed_overall = next( - (m for m in accuracy_metrics if m.tag == ACCURACY_UNPARSED_TAG), None - ) - unparsed_by_task: dict[str, int] = { - m.tag.removeprefix(ACCURACY_UNPARSED_TASK_TAG_PREFIX): int(m.sum or 0) - for m in accuracy_metrics - if m.tag.startswith(ACCURACY_UNPARSED_TASK_TAG_PREFIX) - } - table = Table(title="Accuracy Benchmark Results", show_lines=True) table.add_column("Task", style="cyan", min_width=30) table.add_column("Correct", justify="right") @@ -77,57 +52,44 @@ async def export(self, console: Console) -> None: table.add_column("Unparsed", justify="right", style="yellow") table.add_column("Accuracy", justify="right", style="bold") - for m in task_metrics: - task_name = m.tag.removeprefix(ACCURACY_TASK_TAG_PREFIX) - acc_str = f"{m.current:.2%}" if m.current is not None else "N/A" - unparsed_count = str(unparsed_by_task.get(task_name, 0)) + for task_name, stats in sorted(summary.per_task.items()): table.add_row( task_name, - str(m.sum or 0), - str(m.count or 0), - unparsed_count, - acc_str, + str(stats.passed), + str(stats.total), + str(stats.unparsed), + f"{stats.accuracy_rate:.2%}", ) - if overall: - acc_str = f"{overall.current:.2%}" if overall.current is not None else "N/A" - overall_unparsed = str( - int(unparsed_overall.sum or 0) if unparsed_overall else 0 - ) + if summary.total_evaluated: table.add_row( "[bold]OVERALL[/bold]", - str(overall.sum or 0), - str(overall.count or 0), - overall_unparsed, - f"[bold green]{acc_str}[/bold green]", + str(summary.total_passed), + str(summary.total_evaluated), + str(summary.overall_unparsed), + f"[bold green]{summary.accuracy_rate:.2%}[/bold green]", style="on dark_green", ) console.print() console.print(table) - self._maybe_warn_all_unparsed(console, overall, unparsed_overall) + self._maybe_warn_all_unparsed(console, summary) def _maybe_warn_all_unparsed( self, console: Console, - overall: Any, - unparsed_overall: Any, + summary: AccuracySummary, ) -> None: """Loud-but-actionable diagnostic for the "accuracy=0 because the server, not the model" case. - Triggers when every task reports 100% unparsed responses — almost + Triggers when every evaluated response reports unparsed output — almost always a mock server or misconfigured endpoint, not an accuracy - problem. Does not gate on overall_total so it fires on tiny smoke - runs. + problem. Does not gate on total count so it fires on tiny smoke runs. """ if not ( - overall - and overall.count - and unparsed_overall - and unparsed_overall.sum is not None - and unparsed_overall.count - and int(unparsed_overall.sum) >= int(unparsed_overall.count) + summary.total_evaluated + and summary.overall_unparsed >= summary.total_evaluated ): return # Console-only diagnostic: export() legitimately runs once per target diff --git a/src/aiperf/accuracy/accuracy_data_exporter.py b/src/aiperf/accuracy/accuracy_data_exporter.py index e291936013..8f6613a37b 100644 --- a/src/aiperf/accuracy/accuracy_data_exporter.py +++ b/src/aiperf/accuracy/accuracy_data_exporter.py @@ -8,26 +8,19 @@ from pathlib import Path from typing import Any -from aiperf.accuracy.models import ( - ACCURACY_METRIC_PREFIX, - ACCURACY_OVERALL_TAG, - ACCURACY_TASK_TAG_PREFIX, - ACCURACY_UNPARSED_TAG, - ACCURACY_UNPARSED_TASK_TAG_PREFIX, -) from aiperf.common.exceptions import DataExporterDisabled from aiperf.common.mixins import AIPerfLoggerMixin from aiperf.exporters.exporter_config import ExporterConfig, FileExportInfo -AccuracyCsvRow = tuple[ - str, int, int, int, str -] # (task, correct, total, unparsed, accuracy) +_CSV_COLUMNS = ["task", "total", "passed", "unparsed", "accuracy_rate", "unparsed_rate"] class AccuracyDataExporter(AIPerfLoggerMixin): """Data exporter for accuracy benchmarking results. - Exports per-task accuracy summary to CSV for offline analysis. + Exports the per-task accuracy summary (plus an OVERALL row) to CSV for + offline analysis, sourced from the structured ``AccuracySummary`` delivered + on the dedicated accuracy channel. """ def __init__(self, exporter_config: ExporterConfig, **kwargs: Any) -> None: @@ -51,58 +44,23 @@ def get_export_info(self) -> FileExportInfo: ) async def export(self) -> None: - """Write per-task accuracy summary to CSV at the path from ``get_export_info``. + """Write the per-task accuracy summary to CSV at the path from ``get_export_info``. - Columns: task, correct, total, accuracy (4 decimal places). Rows are - emitted for each ``accuracy.task.*`` metric plus a final OVERALL row. - Does nothing if no ``accuracy.*`` metrics are present in results. + Columns: task, total, passed, unparsed, accuracy_rate, unparsed_rate. + One row per task plus a final OVERALL row. Does nothing when no accuracy + summary was delivered. """ - results = self.exporter_config.results - if results is None or results.records is None: + summary = self.exporter_config.accuracy_results + if summary is None: return - accuracy_metrics = [ - r for r in results.records if r.tag.startswith(ACCURACY_METRIC_PREFIX) - ] - if not accuracy_metrics: - return - - unparsed_overall = next( - (m for m in accuracy_metrics if m.tag == ACCURACY_UNPARSED_TAG), None - ) - unparsed_by_task: dict[str, int] = { - m.tag.removeprefix(ACCURACY_UNPARSED_TASK_TAG_PREFIX): int(m.sum or 0) - for m in accuracy_metrics - if m.tag.startswith(ACCURACY_UNPARSED_TASK_TAG_PREFIX) - } - - rows: list[AccuracyCsvRow] = [] - for m in accuracy_metrics: - if m.tag == ACCURACY_OVERALL_TAG: - task_name = "OVERALL" - unparsed = int(unparsed_overall.sum or 0) if unparsed_overall else 0 - elif m.tag.startswith(ACCURACY_TASK_TAG_PREFIX): - task_name = m.tag.removeprefix(ACCURACY_TASK_TAG_PREFIX) - unparsed = unparsed_by_task.get(task_name, 0) - else: - continue - rows.append( - ( - task_name, - int(m.sum or 0), - int(m.count or 0), - unparsed, - f"{m.current:.4f}" if m.current is not None else "", - ) - ) - + rows = summary.to_csv() await asyncio.to_thread(self._write_csv, rows) self.info(f"Accuracy results exported to {self._csv_path}") - def _write_csv(self, rows: list[AccuracyCsvRow]) -> None: + def _write_csv(self, rows: list[dict[str, Any]]) -> None: self._csv_path.parent.mkdir(parents=True, exist_ok=True) with open(self._csv_path, "w", newline="") as f: - writer = csv.writer(f) - writer.writerow(["task", "correct", "total", "unparsed", "accuracy"]) - for row in rows: - writer.writerow(row) + writer = csv.DictWriter(f, fieldnames=_CSV_COLUMNS) + writer.writeheader() + writer.writerows(rows) diff --git a/src/aiperf/controller/system_controller.py b/src/aiperf/controller/system_controller.py index 6a5e021da9..28dba50814 100644 --- a/src/aiperf/controller/system_controller.py +++ b/src/aiperf/controller/system_controller.py @@ -9,6 +9,7 @@ from rich.console import Console from rich.panel import Panel +from aiperf.accuracy.models import AccuracySummary from aiperf.cli_utils import ( print_developer_mode_warning, warn_accuracy_temperature, @@ -32,6 +33,7 @@ CommandResponse, CommandSuccessResponse, HeartbeatMessage, + ProcessAccuracyResultMessage, ProcessAllResultsMessage, ProcessRecordsResultMessage, ProcessServerMetricsResultMessage, @@ -159,9 +161,15 @@ def __init__( self._exit_errors: list[ExitErrorInfo] = [] self._telemetry_results: TelemetryExportData | None = None self._server_metrics_results: ServerMetricsResults | None = None + self._accuracy_results: AccuracySummary | None = None self._profile_results_received = False self._should_wait_for_telemetry = False self._should_wait_for_server_metrics = False + # Accuracy has no manager status message; gate on config directly since + # RecordsManager publishes the summary iff accuracy is enabled. + self._should_wait_for_accuracy = ( + self.run.cfg.accuracy is not None and self.run.cfg.accuracy.enabled + ) self._shutdown_triggered = False self._shutdown_lock = asyncio.Lock() @@ -709,6 +717,19 @@ async def _on_process_server_metrics_result_message( self._should_wait_for_server_metrics = False await self._check_and_trigger_shutdown() + @on_message(MessageType.PROCESS_ACCURACY_RESULT) + async def _on_process_accuracy_result_message( + self, message: ProcessAccuracyResultMessage + ) -> None: + """Handle an accuracy results message.""" + try: + self._accuracy_results = message.accuracy_result.results + except Exception as e: + self.exception(f"Error processing accuracy results message: {e!r}") + finally: + self._should_wait_for_accuracy = False + await self._check_and_trigger_shutdown() + def _is_api_service_alive(self) -> bool: """Return True iff the API service is registered and its process is live. @@ -761,6 +782,7 @@ async def _check_and_trigger_shutdown(self) -> None: f"_check_and_trigger_shutdown: profile_received={self._profile_results_received}, " f"wait_telemetry={self._should_wait_for_telemetry}, telemetry_results={self._telemetry_results is not None}, " f"wait_server_metrics={self._should_wait_for_server_metrics}, server_metrics_results={self._server_metrics_results is not None}, " + f"wait_accuracy={self._should_wait_for_accuracy}, accuracy_results={self._accuracy_results is not None}, " f"shutdown_triggered={self._shutdown_triggered}" ) # Check if we should trigger shutdown (with lock protection) @@ -788,7 +810,15 @@ async def _check_and_trigger_shutdown(self) -> None: or self._server_metrics_results is not None ) - if telemetry_ready_for_shutdown and server_metrics_ready_for_shutdown: + accuracy_ready_for_shutdown = ( + not self._should_wait_for_accuracy or self._accuracy_results is not None + ) + + if ( + telemetry_ready_for_shutdown + and server_metrics_ready_for_shutdown + and accuracy_ready_for_shutdown + ): self._shutdown_triggered = True should_shutdown = True self.info("All results received, initiating shutdown") @@ -797,6 +827,8 @@ async def _check_and_trigger_shutdown(self) -> None: self.info("Waiting for telemetry results...") if not server_metrics_ready_for_shutdown: self.info("Waiting for server metrics results...") + if not accuracy_ready_for_shutdown: + self.info("Waiting for accuracy results...") # Call stop() OUTSIDE the lock to prevent deadlock if should_shutdown: @@ -1091,6 +1123,7 @@ async def _print_post_benchmark_info_and_metrics(self) -> None: run=self.run, telemetry_results=self._telemetry_results, server_metrics_results=self._server_metrics_results, + accuracy_results=self._accuracy_results, ) # Export data files (CSV, JSON) with complete dataset including telemetry diff --git a/src/aiperf/exporters/exporter_config.py b/src/aiperf/exporters/exporter_config.py index e3fd63f812..50d4cd712c 100644 --- a/src/aiperf/exporters/exporter_config.py +++ b/src/aiperf/exporters/exporter_config.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import TYPE_CHECKING +from aiperf.accuracy.models import AccuracySummary from aiperf.common.models import ProfileResults from aiperf.common.models.export_models import TelemetryExportData from aiperf.common.models.server_metrics_models import ServerMetricsResults @@ -22,6 +23,7 @@ class ExporterConfig: cfg: "BenchmarkConfig" telemetry_results: TelemetryExportData | None server_metrics_results: ServerMetricsResults | None = None + accuracy_results: AccuracySummary | None = None run: "BenchmarkRun | None" = None diff --git a/src/aiperf/exporters/exporter_manager.py b/src/aiperf/exporters/exporter_manager.py index 4e3b690c58..8965622a3c 100644 --- a/src/aiperf/exporters/exporter_manager.py +++ b/src/aiperf/exporters/exporter_manager.py @@ -7,6 +7,7 @@ from rich.console import Console +from aiperf.accuracy.models import AccuracySummary from aiperf.common.environment import Environment from aiperf.common.exceptions import ( ConsoleExporterDisabled, @@ -38,6 +39,7 @@ def __init__( run: "BenchmarkRun", telemetry_results: TelemetryExportData | None, server_metrics_results: ServerMetricsResults | None = None, + accuracy_results: AccuracySummary | None = None, **kwargs, ) -> None: super().__init__(**kwargs) @@ -49,6 +51,7 @@ def __init__( cfg=run.cfg, telemetry_results=telemetry_results, server_metrics_results=server_metrics_results, + accuracy_results=accuracy_results, run=run, ) diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 427eec10e6..1df5d7b666 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -1425,13 +1425,16 @@ async def _process_results( except Exception as e: self.exception(f"Failed to publish server metrics results: {e!r}") - if self.run.cfg.accuracy is None or not self.run.cfg.accuracy.enabled: - self.debug("Accuracy evaluation is disabled, skipping publish") - else: + accuracy_enabled = ( + self.run.cfg.accuracy is not None and self.run.cfg.accuracy.enabled + ) + if accuracy_enabled and phase == CreditPhase.PROFILING: try: await self._publish_accuracy_results(phase) except Exception as e: self.exception(f"Failed to publish accuracy results: {e!r}") + else: + self.debug("Accuracy publish skipped (disabled or non-profiling phase)") # Publish the unified ProcessAllResultsMessage over the populated # accumulators. The per-stream result messages above remain the diff --git a/tests/unit/accuracy/test_accuracy_console_exporter.py b/tests/unit/accuracy/test_accuracy_console_exporter.py index 820613b13d..1ddbf78ff9 100644 --- a/tests/unit/accuracy/test_accuracy_console_exporter.py +++ b/tests/unit/accuracy/test_accuracy_console_exporter.py @@ -8,61 +8,52 @@ from rich.console import Console from aiperf.accuracy.accuracy_console_exporter import AccuracyConsoleExporter -from aiperf.common.models import MetricResult -from aiperf.common.models.record_models import ProfileResults +from aiperf.accuracy.models import AccuracySummary, TaskAccuracyStats from aiperf.exporters.exporter_config import ExporterConfig from aiperf.plugin.enums import AccuracyBenchmarkType, EndpointType from tests.unit.conftest import make_benchmark_run -def _make_exporter(records: list[MetricResult] | None) -> AccuracyConsoleExporter: +def _make_exporter(summary: AccuracySummary | None) -> AccuracyConsoleExporter: cfg = make_benchmark_run( model_names=["test-model"], endpoint_type=EndpointType.COMPLETIONS, streaming=False, accuracy={"benchmark": AccuracyBenchmarkType.MMLU}, ).cfg - results = ( - ProfileResults(records=records, completed=0, start_ns=0, end_ns=1) - if records is not None - else None - ) exporter_config = ExporterConfig( cfg=cfg, - results=results, + results=None, telemetry_results=None, + accuracy_results=summary, ) return AccuracyConsoleExporter(exporter_config=exporter_config) -def _make_metric(tag: str, correct: int, total: int, accuracy: float) -> MetricResult: - return MetricResult( - tag=tag, - header=tag, - unit="ratio", - sum=correct, - count=total, - current=accuracy, +def _task(passed: int, total: int, unparsed: int) -> TaskAccuracyStats: + return TaskAccuracyStats( + total=total, + passed=passed, + unparsed=unparsed, + accuracy_rate=passed / total if total else 0.0, + unparsed_rate=unparsed / total if total else 0.0, ) @pytest.mark.asyncio class TestAccuracyConsoleExporterExport: async def test_prints_table_with_task_and_overall_rows(self) -> None: - exporter = _make_exporter( - records=[ - _make_metric("accuracy.overall", correct=8, total=10, accuracy=0.8), - _make_metric("accuracy.task.algebra", correct=3, total=5, accuracy=0.6), - _make_metric("accuracy.task.history", correct=5, total=5, accuracy=1.0), - _make_metric("accuracy.unparsed", correct=1, total=10, accuracy=0.1), - _make_metric( - "accuracy.unparsed.task.algebra", correct=1, total=5, accuracy=0.2 - ), - _make_metric( - "accuracy.unparsed.task.history", correct=0, total=5, accuracy=0.0 - ), - ] + summary = AccuracySummary( + total_evaluated=10, + total_passed=8, + accuracy_rate=0.8, + overall_unparsed=1, + per_task={ + "algebra": _task(passed=3, total=5, unparsed=1), + "history": _task(passed=5, total=5, unparsed=0), + }, ) + exporter = _make_exporter(summary) buf = io.StringIO() console = Console(file=buf, highlight=False) await exporter.export(console) @@ -73,35 +64,21 @@ async def test_prints_table_with_task_and_overall_rows(self) -> None: assert "OVERALL" in output assert "Unparsed" in output - async def test_no_output_when_results_is_none(self) -> None: - exporter = _make_exporter(records=None) - console = MagicMock() - await exporter.export(console) - console.print.assert_not_called() - - async def test_no_output_when_records_is_none(self) -> None: - exporter = _make_exporter(records=None) - exporter.exporter_config.results = ProfileResults( - records=None, completed=0, start_ns=0, end_ns=1 - ) - console = MagicMock() - await exporter.export(console) - console.print.assert_not_called() - - async def test_no_output_when_no_accuracy_metrics(self) -> None: - exporter = _make_exporter( - records=[_make_metric("throughput", correct=0, total=100, accuracy=0.0)] - ) + async def test_no_output_when_summary_is_none(self) -> None: + exporter = _make_exporter(None) console = MagicMock() await exporter.export(console) console.print.assert_not_called() - async def test_overall_row_omitted_when_no_overall_metric(self) -> None: - exporter = _make_exporter( - records=[ - _make_metric("accuracy.task.algebra", correct=3, total=5, accuracy=0.6), - ] + async def test_overall_row_omitted_when_no_evaluations(self) -> None: + summary = AccuracySummary( + total_evaluated=0, + total_passed=0, + accuracy_rate=0.0, + overall_unparsed=0, + per_task={"algebra": _task(passed=3, total=5, unparsed=0)}, ) + exporter = _make_exporter(summary) buf = io.StringIO() console = Console(file=buf, highlight=False) await exporter.export(console) @@ -111,11 +88,14 @@ async def test_overall_row_omitted_when_no_overall_metric(self) -> None: assert "algebra" in output async def test_accuracy_formatted_as_percentage(self) -> None: - exporter = _make_exporter( - records=[ - _make_metric("accuracy.task.algebra", correct=3, total=5, accuracy=0.6), - ] + summary = AccuracySummary( + total_evaluated=5, + total_passed=3, + accuracy_rate=0.6, + overall_unparsed=0, + per_task={"algebra": _task(passed=3, total=5, unparsed=0)}, ) + exporter = _make_exporter(summary) buf = io.StringIO() console = Console(file=buf, highlight=False) await exporter.export(console) @@ -123,28 +103,18 @@ async def test_accuracy_formatted_as_percentage(self) -> None: assert "60.00%" in buf.getvalue() async def test_warns_when_all_responses_unparsed(self) -> None: - """Smoke-test J regression: when 100% of responses fail to parse, + """Smoke-test regression: when 100% of responses fail to parse, the exporter must surface a loud diagnostic so users do not mistake mock-server / misconfigured-endpoint output for real accuracy=0% results.""" - exporter = _make_exporter( - records=[ - _make_metric("accuracy.overall", correct=0, total=5, accuracy=0.0), - _make_metric( - "accuracy.task.abstract_algebra", - correct=0, - total=5, - accuracy=0.0, - ), - _make_metric("accuracy.unparsed", correct=5, total=5, accuracy=1.0), - _make_metric( - "accuracy.unparsed.task.abstract_algebra", - correct=5, - total=5, - accuracy=1.0, - ), - ] + summary = AccuracySummary( + total_evaluated=5, + total_passed=0, + accuracy_rate=0.0, + overall_unparsed=5, + per_task={"abstract_algebra": _task(passed=0, total=5, unparsed=5)}, ) + exporter = _make_exporter(summary) buf = io.StringIO() console = Console(file=buf, highlight=False) await exporter.export(console) @@ -155,18 +125,16 @@ async def test_warns_when_all_responses_unparsed(self) -> None: assert "inference server" in output async def test_no_warning_when_partial_unparsed(self) -> None: - """Mixed parsed/unparsed runs are normal — the diagnostic must + """Mixed parsed/unparsed runs are normal - the diagnostic must only fire on the 100%-unparsed pathology.""" - exporter = _make_exporter( - records=[ - _make_metric("accuracy.overall", correct=2, total=5, accuracy=0.4), - _make_metric("accuracy.task.algebra", correct=2, total=5, accuracy=0.4), - _make_metric("accuracy.unparsed", correct=2, total=5, accuracy=0.4), - _make_metric( - "accuracy.unparsed.task.algebra", correct=2, total=5, accuracy=0.4 - ), - ] + summary = AccuracySummary( + total_evaluated=5, + total_passed=2, + accuracy_rate=0.4, + overall_unparsed=2, + per_task={"algebra": _task(passed=2, total=5, unparsed=2)}, ) + exporter = _make_exporter(summary) buf = io.StringIO() console = Console(file=buf, highlight=False) await exporter.export(console) diff --git a/tests/unit/accuracy/test_accuracy_data_exporter.py b/tests/unit/accuracy/test_accuracy_data_exporter.py index 6bf56e2f55..f4a0432180 100644 --- a/tests/unit/accuracy/test_accuracy_data_exporter.py +++ b/tests/unit/accuracy/test_accuracy_data_exporter.py @@ -7,108 +7,92 @@ import pytest from aiperf.accuracy.accuracy_data_exporter import AccuracyDataExporter -from aiperf.common.models import MetricResult -from aiperf.common.models.record_models import ProfileResults +from aiperf.accuracy.models import AccuracySummary, TaskAccuracyStats +from aiperf.common.exceptions import DataExporterDisabled from aiperf.exporters.exporter_config import ExporterConfig from aiperf.plugin.enums import AccuracyBenchmarkType, EndpointType from tests.unit.conftest import make_benchmark_run -def _make_cfg(): +def _make_cfg(accuracy: dict | None = None): return make_benchmark_run( model_names=["test-model"], endpoint_type=EndpointType.CHAT, streaming=False, - accuracy={"benchmark": AccuracyBenchmarkType.MMLU}, + accuracy=accuracy, ).cfg -def _make_exporter(tmp_path: Path, records: list[MetricResult]) -> AccuracyDataExporter: +def _make_summary() -> AccuracySummary: + return AccuracySummary( + total_evaluated=10, + total_passed=8, + accuracy_rate=0.8, + overall_unparsed=1, + grader_name="mmlu", + per_task={ + "algebra": TaskAccuracyStats( + total=5, passed=3, unparsed=1, accuracy_rate=0.6, unparsed_rate=0.2 + ), + "history": TaskAccuracyStats( + total=5, passed=5, unparsed=0, accuracy_rate=1.0, unparsed_rate=0.0 + ), + }, + ) + + +def _make_exporter( + tmp_path: Path, summary: AccuracySummary | None +) -> AccuracyDataExporter: exporter_config = ExporterConfig( - cfg=_make_cfg(), - results=ProfileResults( - records=records, - completed=len(records), - start_ns=0, - end_ns=1, - ), + cfg=_make_cfg({"benchmark": AccuracyBenchmarkType.MMLU}), + results=None, telemetry_results=None, + accuracy_results=summary, ) exporter = AccuracyDataExporter(exporter_config=exporter_config) exporter._csv_path = tmp_path / "accuracy_results.csv" return exporter -def _make_metric(tag: str, correct: int, total: int, accuracy: float) -> MetricResult: - return MetricResult( - tag=tag, - header=tag, - unit="ratio", - sum=correct, - count=total, - current=accuracy, - ) - - @pytest.mark.asyncio class TestAccuracyDataExporterExport: - async def test_export_writes_overall_and_task_rows(self, tmp_path: Path) -> None: - records = [ - _make_metric("accuracy.overall", correct=8, total=10, accuracy=0.8), - _make_metric("accuracy.task.algebra", correct=3, total=5, accuracy=0.6), - _make_metric("accuracy.task.history", correct=5, total=5, accuracy=1.0), - _make_metric("accuracy.unparsed", correct=1, total=10, accuracy=0.1), - _make_metric( - "accuracy.unparsed.task.algebra", correct=1, total=5, accuracy=0.2 - ), - _make_metric( - "accuracy.unparsed.task.history", correct=0, total=5, accuracy=0.0 - ), - ] - exporter = _make_exporter(tmp_path, records) + async def test_export_writes_task_rows_and_overall(self, tmp_path: Path) -> None: + exporter = _make_exporter(tmp_path, _make_summary()) await exporter.export() rows = list(csv.reader(exporter._csv_path.open())) - assert rows[0] == ["task", "correct", "total", "unparsed", "accuracy"] - assert rows[1] == ["OVERALL", "8", "10", "1", "0.8000"] - assert rows[2] == ["algebra", "3", "5", "1", "0.6000"] - assert rows[3] == ["history", "5", "5", "0", "1.0000"] - - async def test_export_skips_non_accuracy_metrics(self, tmp_path: Path) -> None: - records = [ - _make_metric("request_latency", correct=0, total=100, accuracy=0.0), - _make_metric("accuracy.overall", correct=4, total=10, accuracy=0.4), + assert rows[0] == [ + "task", + "total", + "passed", + "unparsed", + "accuracy_rate", + "unparsed_rate", ] - exporter = _make_exporter(tmp_path, records) - - await exporter.export() - - rows = list(csv.reader(exporter._csv_path.open())) - assert len(rows) == 2 # header + overall only - assert rows[1][0] == "OVERALL" + # Per-task rows are sorted by name. + assert rows[1] == ["algebra", "5", "3", "1", "0.6", "0.2"] + assert rows[2] == ["history", "5", "5", "0", "1.0", "0.0"] + assert rows[3] == ["OVERALL", "10", "8", "1", "0.8", "0.1"] - async def test_export_does_nothing_when_no_accuracy_metrics( + async def test_export_does_nothing_when_summary_is_none( self, tmp_path: Path ) -> None: - records = [_make_metric("request_latency", correct=0, total=10, accuracy=0.0)] - exporter = _make_exporter(tmp_path, records) + exporter = _make_exporter(tmp_path, None) await exporter.export() assert not exporter._csv_path.exists() - async def test_export_does_nothing_when_records_is_none( + async def test_constructor_raises_when_accuracy_disabled( self, tmp_path: Path ) -> None: exporter_config = ExporterConfig( - cfg=_make_cfg(), - results=ProfileResults(records=None, completed=0, start_ns=0, end_ns=1), + cfg=_make_cfg(None), + results=None, telemetry_results=None, + accuracy_results=_make_summary(), ) - exporter = AccuracyDataExporter(exporter_config=exporter_config) - exporter._csv_path = tmp_path / "accuracy_results.csv" - - await exporter.export() - - assert not exporter._csv_path.exists() + with pytest.raises(DataExporterDisabled): + AccuracyDataExporter(exporter_config=exporter_config) From bc766fcf0b6fa3c1c5b41eb1d3ac1ca8208f8105 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 20:36:13 -0700 Subject: [PATCH 15/39] fix(records): guarantee exactly-once accuracy result publish; test shutdown gate Ensure _publish_accuracy_results always publishes exactly one ProcessAccuracyResultMessage (terminal results=None on no-accumulator or export failure) so SystemController's _should_wait_for_accuracy always clears and shutdown cannot hang. Add SystemController accuracy shutdown-gate tests and a console-exporter disabled-raises test. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/records/records_manager.py | 25 ++- .../test_accuracy_console_exporter.py | 17 ++ .../controller/test_accuracy_shutdown_gate.py | 175 ++++++++++++++++++ .../records/test_records_manager_accuracy.py | 41 +++- 4 files changed, 250 insertions(+), 8 deletions(-) create mode 100644 tests/unit/controller/test_accuracy_shutdown_gate.py diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 1df5d7b666..51cebbdd24 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any -from aiperf.accuracy.models import ProcessAccuracyResult +from aiperf.accuracy.models import AccuracySummary, ProcessAccuracyResult from aiperf.common.accumulator_protocols import ( AccumulatorProtocol, ExportContext, @@ -1501,12 +1501,25 @@ async def _publish_accuracy_results(self, phase: CreditPhase) -> None: Mirrors ``_publish_telemetry_results``: exports the phase-scoped accuracy summary from the AccuracyAccumulator and publishes it independently from inference results. + + Exactly-once contract: this method only runs when accuracy is enabled and + the phase is PROFILING, and it ALWAYS publishes exactly one + ``ProcessAccuracyResultMessage`` before returning. The SystemController + clears ``_should_wait_for_accuracy`` only on receipt of that message, so a + missing publish would hang shutdown forever. When there is no accumulator, + or ``export_results`` raises, a terminal ``results=None`` summary is + published instead; the success path publishes the real summary. Every path + publishes once and only once. """ - if self._accuracy_accumulator is None: - return - summary = await self._accuracy_accumulator.export_results( - ExportContext(phase=phase) - ) + summary: AccuracySummary | None = None + if self._accuracy_accumulator is not None: + try: + summary = await self._accuracy_accumulator.export_results( + ExportContext(phase=phase) + ) + except Exception as e: # noqa: BLE001 - must still publish a terminal message + self.exception(f"Accuracy summary export failed: {e!r}") + summary = None await self.publish( ProcessAccuracyResultMessage( service_id=self.service_id, diff --git a/tests/unit/accuracy/test_accuracy_console_exporter.py b/tests/unit/accuracy/test_accuracy_console_exporter.py index 1ddbf78ff9..e4cbe8856c 100644 --- a/tests/unit/accuracy/test_accuracy_console_exporter.py +++ b/tests/unit/accuracy/test_accuracy_console_exporter.py @@ -9,6 +9,7 @@ from aiperf.accuracy.accuracy_console_exporter import AccuracyConsoleExporter from aiperf.accuracy.models import AccuracySummary, TaskAccuracyStats +from aiperf.common.exceptions import ConsoleExporterDisabled from aiperf.exporters.exporter_config import ExporterConfig from aiperf.plugin.enums import AccuracyBenchmarkType, EndpointType from tests.unit.conftest import make_benchmark_run @@ -142,3 +143,19 @@ async def test_no_warning_when_partial_unparsed(self) -> None: output = buf.getvalue() assert "Warning" not in output assert "inference server" not in output + + async def test_constructor_raises_when_accuracy_disabled(self) -> None: + cfg = make_benchmark_run( + model_names=["test-model"], + endpoint_type=EndpointType.COMPLETIONS, + streaming=False, + accuracy=None, + ).cfg + exporter_config = ExporterConfig( + cfg=cfg, + results=None, + telemetry_results=None, + accuracy_results=None, + ) + with pytest.raises(ConsoleExporterDisabled): + AccuracyConsoleExporter(exporter_config=exporter_config) diff --git a/tests/unit/controller/test_accuracy_shutdown_gate.py b/tests/unit/controller/test_accuracy_shutdown_gate.py new file mode 100644 index 0000000000..7bc35b7df5 --- /dev/null +++ b/tests/unit/controller/test_accuracy_shutdown_gate.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the SystemController accuracy shutdown-gate. + +The controller sets ``_should_wait_for_accuracy`` from config at construction and +clears it only when a ``ProcessAccuracyResultMessage`` arrives. While the flag is +True and no accuracy summary has been received, ``_check_and_trigger_shutdown`` +must NOT trigger shutdown; once the message handler runs (with a real summary OR a +terminal ``results=None``), the flag clears and the accuracy term stops blocking. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import pytest + +from aiperf.accuracy.models import AccuracySummary, ProcessAccuracyResult +from aiperf.common.messages import ProcessAccuracyResultMessage +from aiperf.controller.system_controller import SystemController +from aiperf.plugin.enums import AccuracyBenchmarkType + + +def _build_controller(benchmark_run, mock_service_manager, *, accuracy: bool): + """Construct a SystemController mirroring the shared fixture, with accuracy + optionally enabled at construction time so ``_should_wait_for_accuracy`` is + computed by the real ``__init__`` logic rather than being set after the fact. + """ + if accuracy: + from aiperf.config.accuracy import AccuracyConfig + + benchmark_run.cfg.accuracy = AccuracyConfig( + benchmark=AccuracyBenchmarkType.MMLU + ) + + mock_ui = AsyncMock() + mock_comm = AsyncMock() + + def mock_get_class(protocol, name): + if protocol == "service_manager": + return lambda **kwargs: mock_service_manager + if protocol == "ui": + return lambda **kwargs: mock_ui + if protocol == "communication": + return lambda **kwargs: mock_comm + raise ValueError(f"Unknown protocol: {protocol}") + + with ( + patch( + "aiperf.controller.system_controller.plugins.get_class", + side_effect=mock_get_class, + ), + patch("aiperf.controller.system_controller.ProxyManager") as mock_proxy, + patch( + "aiperf.common.mixins.communication_mixin.plugins.get_class", + side_effect=mock_get_class, + ), + ): # fmt: skip + mock_proxy.return_value = AsyncMock() + controller = SystemController(run=benchmark_run, service_id="test_controller") + + controller.stop = AsyncMock() + return controller + + +def _summary() -> AccuracySummary: + return AccuracySummary( + total_evaluated=4, + total_passed=3, + accuracy_rate=0.75, + overall_unparsed=0, + grader_name="multiple_choice", + ) + + +class TestAccuracyShutdownGateEnabled: + """Accuracy ENABLED: the flag blocks shutdown until the message clears it.""" + + @pytest.mark.asyncio + async def test_startup_sets_wait_flag_true( + self, benchmark_run, mock_service_manager + ) -> None: + controller = _build_controller( + benchmark_run, mock_service_manager, accuracy=True + ) + assert controller._should_wait_for_accuracy is True + + @pytest.mark.asyncio + async def test_gate_blocks_shutdown_while_waiting( + self, benchmark_run, mock_service_manager + ) -> None: + controller = _build_controller( + benchmark_run, mock_service_manager, accuracy=True + ) + # Profile results present so only the accuracy term can gate. + controller._profile_results_received = True + controller._accuracy_results = None + + await controller._check_and_trigger_shutdown() + + assert controller._shutdown_triggered is False + controller.stop.assert_not_called() + + @pytest.mark.asyncio + async def test_summary_message_clears_flag_and_unblocks( + self, benchmark_run, mock_service_manager + ) -> None: + controller = _build_controller( + benchmark_run, mock_service_manager, accuracy=True + ) + controller._profile_results_received = True + + summary = _summary() + await controller._on_process_accuracy_result_message( + ProcessAccuracyResultMessage( + service_id="rm", + accuracy_result=ProcessAccuracyResult(results=summary), + ) + ) + + assert controller._should_wait_for_accuracy is False + assert controller._accuracy_results == summary + assert controller._shutdown_triggered is True + controller.stop.assert_awaited_once() + + @pytest.mark.asyncio + async def test_terminal_none_message_clears_flag_and_unblocks( + self, benchmark_run, mock_service_manager + ) -> None: + """A ``results=None`` terminal message must still clear the wait flag so a + summary that could not be computed does not hang shutdown forever.""" + controller = _build_controller( + benchmark_run, mock_service_manager, accuracy=True + ) + controller._profile_results_received = True + + await controller._on_process_accuracy_result_message( + ProcessAccuracyResultMessage( + service_id="rm", + accuracy_result=ProcessAccuracyResult(results=None), + ) + ) + + assert controller._should_wait_for_accuracy is False + assert controller._accuracy_results is None + assert controller._shutdown_triggered is True + controller.stop.assert_awaited_once() + + +class TestAccuracyShutdownGateDisabled: + """Accuracy DISABLED: the accuracy term never blocks shutdown.""" + + @pytest.mark.asyncio + async def test_startup_wait_flag_false( + self, benchmark_run, mock_service_manager + ) -> None: + controller = _build_controller( + benchmark_run, mock_service_manager, accuracy=False + ) + assert controller._should_wait_for_accuracy is False + + @pytest.mark.asyncio + async def test_accuracy_term_never_blocks( + self, benchmark_run, mock_service_manager + ) -> None: + controller = _build_controller( + benchmark_run, mock_service_manager, accuracy=False + ) + controller._profile_results_received = True + controller._accuracy_results = None + + await controller._check_and_trigger_shutdown() + + assert controller._shutdown_triggered is True + controller.stop.assert_awaited_once() diff --git a/tests/unit/records/test_records_manager_accuracy.py b/tests/unit/records/test_records_manager_accuracy.py index c4ee7c9427..c2ba8ed932 100644 --- a/tests/unit/records/test_records_manager_accuracy.py +++ b/tests/unit/records/test_records_manager_accuracy.py @@ -115,8 +115,15 @@ async def test_publishes_none_summary(self) -> None: assert msg.accuracy_result.results is None @pytest.mark.asyncio - async def test_no_accumulator_is_noop(self) -> None: + async def test_no_accumulator_publishes_terminal_none(self) -> None: + """No accumulator must still publish exactly one terminal ``results=None``. + + The SystemController clears ``_should_wait_for_accuracy`` only on this + message, so a bare early-return would hang shutdown forever when the + accumulator failed to construct while accuracy is config-enabled. + """ mgr = MagicMock() + mgr.service_id = "rm" mgr.publish = AsyncMock() mgr._accuracy_accumulator = None mgr._publish_accuracy_results = ( @@ -125,4 +132,34 @@ async def test_no_accumulator_is_noop(self) -> None: await mgr._publish_accuracy_results(CreditPhase.PROFILING) - mgr.publish.assert_not_awaited() + mgr.publish.assert_awaited_once() + msg = mgr.publish.await_args.args[0] + assert isinstance(msg, ProcessAccuracyResultMessage) + assert msg.accuracy_result.results is None + + @pytest.mark.asyncio + async def test_export_raises_still_publishes_terminal_none(self) -> None: + """An ``export_results`` failure logs and still publishes ``results=None``. + + Guarantees the exactly-once terminal message so shutdown never hangs on a + summary-computation error. + """ + accumulator = MagicMock() + accumulator.export_results = AsyncMock(side_effect=RuntimeError("boom")) + + mgr = MagicMock() + mgr.service_id = "rm" + mgr.publish = AsyncMock() + mgr.exception = MagicMock() + mgr._accuracy_accumulator = accumulator + mgr._publish_accuracy_results = ( + RecordsManager._publish_accuracy_results.__get__(mgr) + ) + + await mgr._publish_accuracy_results(CreditPhase.PROFILING) + + assert mgr.exception.called + mgr.publish.assert_awaited_once() + msg = mgr.publish.await_args.args[0] + assert isinstance(msg, ProcessAccuracyResultMessage) + assert msg.accuracy_result.results is None From 17868b41c26f6aa9dda099787f6d78c02347704b Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 20:42:24 -0700 Subject: [PATCH 16/39] refactor(accuracy): remove dead metric_records shoehorn The dedicated accuracy channel (Tasks A-D) fully supersedes the old path. Delete the fake accuracy_correct/accuracy_unparsed aggregate metrics, the AccuracyResultsProcessor accumulator riding metric_records, and the dead accuracy. tag constants/helpers. Regenerate plugin enums (drops AccumulatorType.ACCURACY_RESULTS) and prune obsolete accumulator tests. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- .../accuracy/accuracy_results_processor.py | 256 ------------------ src/aiperf/metrics/types/accuracy_metrics.py | 72 ----- ...st_accuracy_results_processor_summarize.py | 203 -------------- 3 files changed, 531 deletions(-) delete mode 100644 src/aiperf/accuracy/accuracy_results_processor.py delete mode 100644 src/aiperf/metrics/types/accuracy_metrics.py delete mode 100644 tests/unit/accuracy/test_accuracy_results_processor_summarize.py diff --git a/src/aiperf/accuracy/accuracy_results_processor.py b/src/aiperf/accuracy/accuracy_results_processor.py deleted file mode 100644 index a2db4465d8..0000000000 --- a/src/aiperf/accuracy/accuracy_results_processor.py +++ /dev/null @@ -1,256 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -from collections import defaultdict -from typing import TYPE_CHECKING, Any - -from aiperf.accuracy.models import ( - ACCURACY_OVERALL_TAG, - ACCURACY_RECORD_CORRECT_KEY, - ACCURACY_RECORD_UNPARSED_KEY, - ACCURACY_UNPARSED_TAG, - accuracy_task_tag, - accuracy_unparsed_task_tag, -) -from aiperf.common.enums import CreditPhase, MetricConsoleGroup -from aiperf.common.exceptions import PostProcessorDisabled -from aiperf.common.mixins import AIPerfLifecycleMixin -from aiperf.common.models import MetricResult - -if TYPE_CHECKING: - from aiperf.common.accumulator_protocols import ExportContext - from aiperf.common.messages.inference_messages import MetricRecordsData - from aiperf.common.models.dataset_models import DatasetMetadata - from aiperf.config.resolution.plan import BenchmarkRun - - -class AccuracyResultsProcessor(AIPerfLifecycleMixin): - """Results processor for accuracy benchmarking. - - Receives task names via on_dataset_configured (called by RecordsManager - when DatasetConfiguredNotification arrives). Accumulates per-record grading - results from AccuracyRecordProcessor, then summarizes into per-task and - overall accuracy MetricResult objects. - - Counts are keyed by ``CreditPhase`` so that ``export_results(ctx)`` can scope - a summary to a single phase (e.g. PROFILING), matching how MetricsAccumulator - is phase-scoped. Without this, warmup grades would leak into the profiling - accuracy summary (and vice versa) since every record increments the counters. - """ - - # RecordsManager routes phase-scoped-export accumulators through - # export_results(ctx) instead of the unscoped summarize(). - supports_phase_scoped_export = True - - def __init__(self, run: BenchmarkRun, **kwargs: Any) -> None: - acc_cfg = run.cfg.accuracy - if acc_cfg is None or not acc_cfg.enabled: - raise PostProcessorDisabled( - "Accuracy results processor is disabled: accuracy mode is not enabled" - ) - - super().__init__(**kwargs) - self.run = run - - self._tasks: list[str] | None = None - # Per-phase counts: phase -> task -> count. Overall is a phase -> count. - self._task_correct: dict[CreditPhase, dict[str, int]] = defaultdict( - lambda: defaultdict(int) - ) - self._task_total: dict[CreditPhase, dict[str, int]] = defaultdict( - lambda: defaultdict(int) - ) - self._task_unparsed: dict[CreditPhase, dict[str, int]] = defaultdict( - lambda: defaultdict(int) - ) - self._overall_correct: dict[CreditPhase, int] = defaultdict(int) - self._overall_total: dict[CreditPhase, int] = defaultdict(int) - self._overall_unparsed: dict[CreditPhase, int] = defaultdict(int) - - def on_dataset_configured(self, metadata: DatasetMetadata) -> None: - """Receive task names from the DatasetConfiguredNotification. - - Called by RecordsManager before any records are processed. Builds the - ordered list of task names from ConversationMetadata so that - process_record can bucket records without re-loading the benchmark. - """ - self._tasks = [ - c.accuracy_task - for c in metadata.conversations - if c.accuracy_task is not None - ] - - async def process_record(self, record_data: MetricRecordsData) -> None: - """Accumulate per-task accuracy counts from a single record's metrics. - - Reads ``accuracy_correct`` from ``record_data.metrics`` (produced by - AccuracyRecordProcessor) and increments per-task and overall counters. - Records missing the ``accuracy_correct`` key are silently skipped. - - Raises: - RuntimeError: if on_dataset_configured was not called before processing. - """ - if self._tasks is None: - raise RuntimeError( - "AccuracyResultsProcessor: dataset not configured; " - "on_dataset_configured must be called before process_record" - ) - metrics = record_data.metrics - correct = metrics.get(ACCURACY_RECORD_CORRECT_KEY) - if correct is None: - return - - phase = record_data.metadata.benchmark_phase - task = self._tasks[record_data.metadata.session_num % len(self._tasks)] - is_correct = float(correct) >= 0.5 - is_unparsed = float(metrics.get(ACCURACY_RECORD_UNPARSED_KEY, 0.0)) >= 0.5 - - self._overall_total[phase] += 1 - if is_correct: - self._overall_correct[phase] += 1 - if is_unparsed: - self._overall_unparsed[phase] += 1 - - self._task_total[phase][task] += 1 - if is_correct: - self._task_correct[phase][task] += 1 - if is_unparsed: - self._task_unparsed[phase][task] += 1 - - def _phase_scoped_counts( - self, phase: CreditPhase | None - ) -> tuple[int, int, int, dict[str, int], dict[str, int], dict[str, int]]: - """Collapse the per-phase counters into a single scope. - - ``phase`` selects one phase; ``None`` sums across every phase (the - phase-agnostic full-range view). Returns - ``(overall_correct, overall_total, overall_unparsed, - task_correct, task_total, task_unparsed)``. - """ - phases = [phase] if phase is not None else list(self._overall_total.keys()) - - overall_correct = sum(self._overall_correct.get(p, 0) for p in phases) - overall_total = sum(self._overall_total.get(p, 0) for p in phases) - overall_unparsed = sum(self._overall_unparsed.get(p, 0) for p in phases) - - task_correct: dict[str, int] = defaultdict(int) - task_total: dict[str, int] = defaultdict(int) - task_unparsed: dict[str, int] = defaultdict(int) - for p in phases: - for task, count in self._task_total.get(p, {}).items(): - task_total[task] += count - for task, count in self._task_correct.get(p, {}).items(): - task_correct[task] += count - for task, count in self._task_unparsed.get(p, {}).items(): - task_unparsed[task] += count - - return ( - overall_correct, - overall_total, - overall_unparsed, - task_correct, - task_total, - task_unparsed, - ) - - def _build_results(self, phase: CreditPhase | None) -> list[MetricResult]: - """Build accuracy MetricResults scoped to ``phase`` (or all phases). - - Emits: - - ``accuracy.overall``: overall correct/total ratio - - ``accuracy.task.``: per-task correct/total ratio (sorted alphabetically) - - ``accuracy.unparsed``: overall count of responses that required regex fallback - - ``accuracy.unparsed.task.``: per-task unparsed counts (sorted alphabetically) - - Returns an empty list if no records were processed in scope. - """ - ( - overall_correct, - overall_total, - overall_unparsed, - task_correct, - task_total, - task_unparsed, - ) = self._phase_scoped_counts(phase) - - results: list[MetricResult] = [] - - if overall_total > 0: - results.append( - MetricResult( - tag=ACCURACY_OVERALL_TAG, - header="Accuracy (Overall)", - unit="ratio", - count=overall_total, - current=overall_correct / overall_total, - sum=overall_correct, - # Rendered by the dedicated Accuracy Benchmark Results table, - # not the main LLM metrics table (a ratio has no avg/p99/etc, - # so it would show as a row of N/A there). - console_group=MetricConsoleGroup.NONE, - ) - ) - - for task in sorted(task_total.keys()): - total = task_total[task] - correct = task_correct.get(task, 0) - results.append( - MetricResult( - tag=accuracy_task_tag(task), - header=f"Accuracy ({task})", - unit="ratio", - count=total, - current=correct / total if total > 0 else 0.0, - sum=correct, - console_group=MetricConsoleGroup.NONE, - ) - ) - - if overall_total > 0: - results.append( - MetricResult( - tag=ACCURACY_UNPARSED_TAG, - header="Accuracy Unparsed (Overall)", - unit="ratio", - count=overall_total, - current=overall_unparsed / overall_total, - sum=overall_unparsed, - console_group=MetricConsoleGroup.NONE, - ) - ) - - for task in sorted(task_total.keys()): - total = task_total[task] - unparsed = task_unparsed.get(task, 0) - results.append( - MetricResult( - tag=accuracy_unparsed_task_tag(task), - header=f"Accuracy Unparsed ({task})", - unit="ratio", - count=total, - current=unparsed / total if total > 0 else 0.0, - sum=unparsed, - console_group=MetricConsoleGroup.NONE, - ) - ) - - return results - - async def summarize(self) -> list[MetricResult]: - """Return phase-agnostic (all-phase) accuracy and unparsed counts. - - Prefer ``export_results(ctx)`` for phase-scoped summaries; this full-range - view is retained for phase-agnostic callers. - """ - return self._build_results(phase=None) - - async def export_results(self, ctx: ExportContext) -> list[MetricResult]: - """Return accuracy counts scoped to ``ctx.phase`` (all phases if None). - - RecordsManager summarizes profiling and warmup separately; scoping here - keeps warmup grades out of the profiling accuracy summary (and vice - versa), matching MetricsAccumulator's phase-scoped export. - """ - return self._build_results(phase=ctx.phase) diff --git a/src/aiperf/metrics/types/accuracy_metrics.py b/src/aiperf/metrics/types/accuracy_metrics.py deleted file mode 100644 index aa2c52a044..0000000000 --- a/src/aiperf/metrics/types/accuracy_metrics.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from aiperf.accuracy.models import ( - ACCURACY_RECORD_CORRECT_KEY, - ACCURACY_RECORD_UNPARSED_KEY, -) -from aiperf.common.enums import GenericMetricUnit, MetricConsoleGroup, MetricFlags -from aiperf.common.exceptions import NoMetricValue -from aiperf.common.models import ParsedResponseRecord -from aiperf.metrics.base_aggregate_metric import BaseAggregateMetric -from aiperf.metrics.metric_dicts import MetricRecordDict - - -class AccuracyCorrectSumMetric(BaseAggregateMetric[float]): - """Registration for the per-record ``accuracy_correct`` transport key. - - AccuracyRecordProcessor writes ``accuracy_correct`` (1.0 correct / 0.0 - incorrect) into every record's MetricRecordDict to hand the grade to - AccuracyResultsProcessor. MetricsAccumulator only ingests registered tags - (it warns and drops anything else), so this exists purely to register it. - - It is a transport key, NOT the accuracy display: ``INTERNAL`` + - ``console_group=NONE`` keep it out of the console table and file exports. - The user-facing accuracy summary is produced separately by - AccuracyResultsProcessor.summarize() under the ``accuracy.`` (dot) namespace, - which this ``accuracy_`` (underscore) tag deliberately stays out of so the - two can never collide. - """ - - tag = ACCURACY_RECORD_CORRECT_KEY - header = "Accuracy Correct" - unit = GenericMetricUnit.RATIO - flags = MetricFlags.INTERNAL - console_group = MetricConsoleGroup.NONE - required_metrics = None - - def _parse_record( - self, record: ParsedResponseRecord, record_metrics: MetricRecordDict - ) -> float: - value = record_metrics.get(ACCURACY_RECORD_CORRECT_KEY) - if value is None: - raise NoMetricValue(f"{ACCURACY_RECORD_CORRECT_KEY} not in record_metrics") - return float(value) - - -class AccuracyUnparsedSumMetric(BaseAggregateMetric[float]): - """Registration for the per-record ``accuracy_unparsed`` transport key. - - Same role as AccuracyCorrectSumMetric: registers the per-record - ``accuracy_unparsed`` (1.0 when the grader could not cleanly extract the - answer) transport key so MetricsAccumulator accepts it. Transport only -- - ``INTERNAL`` + ``console_group=NONE``; the displayed unparsed counts come - from AccuracyResultsProcessor.summarize() in the ``accuracy.`` namespace. - Registering this under the dot tag ``accuracy.unparsed`` used to collide - with that summary tag and drop the overall unparsed count. - """ - - tag = ACCURACY_RECORD_UNPARSED_KEY - header = "Accuracy Unparsed" - unit = GenericMetricUnit.RATIO - flags = MetricFlags.INTERNAL - console_group = MetricConsoleGroup.NONE - required_metrics = None - - def _parse_record( - self, record: ParsedResponseRecord, record_metrics: MetricRecordDict - ) -> float: - value = record_metrics.get(ACCURACY_RECORD_UNPARSED_KEY) - if value is None: - raise NoMetricValue(f"{ACCURACY_RECORD_UNPARSED_KEY} not in record_metrics") - return float(value) diff --git a/tests/unit/accuracy/test_accuracy_results_processor_summarize.py b/tests/unit/accuracy/test_accuracy_results_processor_summarize.py deleted file mode 100644 index d44c68bab0..0000000000 --- a/tests/unit/accuracy/test_accuracy_results_processor_summarize.py +++ /dev/null @@ -1,203 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import pytest - -from aiperf.accuracy.accuracy_results_processor import AccuracyResultsProcessor -from aiperf.common.accumulator_protocols import ExportContext -from aiperf.common.enums import CreditPhase -from aiperf.plugin.enums import AccuracyBenchmarkType, EndpointType -from tests.unit.conftest import make_benchmark_run - - -def _make_processor() -> AccuracyResultsProcessor: - return AccuracyResultsProcessor( - run=make_benchmark_run( - model_names=["test-model"], - endpoint_type=EndpointType.COMPLETIONS, - streaming=False, - accuracy={"benchmark": AccuracyBenchmarkType.MMLU}, - ) - ) - - -def _seed( - processor: AccuracyResultsProcessor, - *, - phase: CreditPhase = CreditPhase.PROFILING, - overall_total: int = 0, - overall_correct: int = 0, - overall_unparsed: int = 0, - task_total: dict[str, int] | None = None, - task_correct: dict[str, int] | None = None, - task_unparsed: dict[str, int] | None = None, -) -> None: - """Seed the per-phase counters for a single phase.""" - processor._overall_total[phase] = overall_total - processor._overall_correct[phase] = overall_correct - processor._overall_unparsed[phase] = overall_unparsed - for task, count in (task_total or {}).items(): - processor._task_total[phase][task] = count - for task, count in (task_correct or {}).items(): - processor._task_correct[phase][task] = count - for task, count in (task_unparsed or {}).items(): - processor._task_unparsed[phase][task] = count - - -@pytest.mark.asyncio -class TestAccuracyResultsProcessorSummarize: - async def test_empty_returns_no_results(self) -> None: - processor = _make_processor() - results = await processor.summarize() - assert results == [] - - async def test_overall_metric_values(self) -> None: - processor = _make_processor() - _seed(processor, overall_total=10, overall_correct=7) - - results = await processor.summarize() - - overall = next(r for r in results if r.tag == "accuracy.overall") - assert overall.current == pytest.approx(0.7) - assert overall.count == 10 - assert overall.sum == 7 - assert overall.unit == "ratio" - - async def test_task_metrics_sorted_alphabetically(self) -> None: - processor = _make_processor() - _seed( - processor, - overall_total=4, - overall_correct=3, - task_total={"zebra": 2, "algebra": 2}, - task_correct={"zebra": 1, "algebra": 2}, - ) - - results = await processor.summarize() - task_results = [r for r in results if r.tag.startswith("accuracy.task.")] - - assert task_results[0].tag == "accuracy.task.algebra" - assert task_results[1].tag == "accuracy.task.zebra" - - async def test_task_metric_accuracy_calculation(self) -> None: - processor = _make_processor() - _seed( - processor, - overall_total=5, - overall_correct=3, - task_total={"math": 5}, - task_correct={"math": 3}, - ) - - results = await processor.summarize() - - task = next(r for r in results if r.tag == "accuracy.task.math") - assert task.current == pytest.approx(0.6) - assert task.count == 5 - assert task.sum == 3 - assert task.header == "Accuracy (math)" - - async def test_overall_not_emitted_when_no_results_processed(self) -> None: - processor = _make_processor() - _seed(processor, task_total={"math": 3}, task_correct={"math": 2}) - - results = await processor.summarize() - - tags = [r.tag for r in results] - assert "accuracy.overall" not in tags - assert "accuracy.unparsed" not in tags - assert "accuracy.task.math" in tags - - async def test_multiple_tasks_each_get_own_metric(self) -> None: - processor = _make_processor() - _seed( - processor, - overall_total=6, - overall_correct=4, - task_total={"history": 2, "biology": 2, "physics": 2}, - task_correct={"history": 1, "biology": 1, "physics": 1}, - ) - - results = await processor.summarize() - task_tags = {r.tag for r in results if r.tag.startswith("accuracy.task.")} - - assert task_tags == { - "accuracy.task.history", - "accuracy.task.biology", - "accuracy.task.physics", - } - - async def test_unparsed_overall_emitted_when_records_processed(self) -> None: - processor = _make_processor() - _seed(processor, overall_total=10, overall_correct=7, overall_unparsed=3) - - results = await processor.summarize() - - unparsed = next(r for r in results if r.tag == "accuracy.unparsed") - assert unparsed.sum == 3 - assert unparsed.count == 10 - assert unparsed.current == pytest.approx(0.3) - - async def test_unparsed_per_task_emitted(self) -> None: - processor = _make_processor() - _seed( - processor, - overall_total=5, - overall_correct=3, - task_total={"math": 5}, - task_correct={"math": 3}, - task_unparsed={"math": 2}, - ) - - results = await processor.summarize() - - unparsed_task = next( - r for r in results if r.tag == "accuracy.unparsed.task.math" - ) - assert unparsed_task.sum == 2 - assert unparsed_task.count == 5 - assert unparsed_task.current == pytest.approx(0.4) - - async def test_unparsed_zero_when_all_conforming(self) -> None: - processor = _make_processor() - _seed( - processor, - overall_total=5, - overall_correct=5, - task_total={"math": 5}, - task_correct={"math": 5}, - ) - - results = await processor.summarize() - - unparsed = next(r for r in results if r.tag == "accuracy.unparsed") - assert unparsed.sum == 0 - assert unparsed.current == pytest.approx(0.0) - - async def test_export_results_scopes_to_phase(self) -> None: - """export_results(ctx) must isolate the requested phase so warmup grades - do not leak into the profiling accuracy summary (and vice versa).""" - processor = _make_processor() - # One correct warmup record, one incorrect profiling record. - _seed(processor, phase=CreditPhase.WARMUP, overall_total=1, overall_correct=1) - _seed( - processor, phase=CreditPhase.PROFILING, overall_total=1, overall_correct=0 - ) - - profiling = await processor.export_results( - ExportContext(phase=CreditPhase.PROFILING) - ) - overall = next(r for r in profiling if r.tag == "accuracy.overall") - assert overall.count == 1 - assert overall.current == pytest.approx(0.0) - - warmup = await processor.export_results(ExportContext(phase=CreditPhase.WARMUP)) - overall = next(r for r in warmup if r.tag == "accuracy.overall") - assert overall.count == 1 - assert overall.current == pytest.approx(1.0) - - # summarize() (phase-agnostic) still sees both records combined. - combined = await processor.summarize() - overall = next(r for r in combined if r.tag == "accuracy.overall") - assert overall.count == 2 - assert overall.current == pytest.approx(0.5) From 08445970926a9cc251cf22255e8cf2db4519fa1e Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 20:42:37 -0700 Subject: [PATCH 17/39] refactor(accuracy): drop dead accuracy tag constants and stale refs Remove the nine dead accuracy. tag constants/helpers from models.py, the accuracy_results plugins.yaml entry, regenerated enums, and fix stale AccuracyResultsProcessor mentions in records_manager/dataset-loader docstrings plus the obsolete accumulator tests. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/accuracy/models.py | 28 ---- .../dataset/loader/accuracy_dataset_loader.py | 8 +- src/aiperf/plugin/enums.py | 2 +- src/aiperf/plugin/plugins.yaml | 8 -- src/aiperf/records/records_manager.py | 4 +- .../test_accuracy_record_processor.py | 135 ------------------ 6 files changed, 8 insertions(+), 177 deletions(-) diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index a4e2782b71..220903b049 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -11,34 +11,6 @@ from aiperf.common.enums import CreditPhase from aiperf.common.models.base_models import AIPerfBaseModel -# Summary/metric tags (the ``accuracy.`` dot namespace) emitted by -# AccuracyResultsProcessor.summarize() and read by the accuracy exporters. -ACCURACY_OVERALL_TAG = "accuracy.overall" -ACCURACY_TASK_TAG_PREFIX = "accuracy.task." -ACCURACY_UNPARSED_TAG = "accuracy.unparsed" -ACCURACY_UNPARSED_TASK_TAG_PREFIX = "accuracy.unparsed.task." -ACCURACY_METRIC_PREFIX = "accuracy." - -# Per-record TRANSPORT keys (NOT metric tags): AccuracyRecordProcessor writes -# these into each record's MetricRecordDict to hand the grade to -# AccuracyResultsProcessor.process_record. They use an ``accuracy_`` (underscore) -# prefix so they live outside the ``accuracy.`` (dot) summary namespace and can -# never be mistaken for -- or collide with -- a summary/metric tag. Keeping them -# in the dot namespace is what let the registered ``accuracy.unparsed`` transport -# metric shadow the ``accuracy.unparsed`` summary tag; the underscore separates them. -ACCURACY_RECORD_CORRECT_KEY = "accuracy_correct" -ACCURACY_RECORD_UNPARSED_KEY = "accuracy_unparsed" - - -def accuracy_task_tag(task: str) -> str: - """Build the MetricResult.tag for a per-task accuracy result.""" - return f"{ACCURACY_TASK_TAG_PREFIX}{task}" - - -def accuracy_unparsed_task_tag(task: str) -> str: - """Build the MetricResult.tag for a per-task unparsed-count result.""" - return f"{ACCURACY_UNPARSED_TASK_TAG_PREFIX}{task}" - class AccuracyChatMessage(TypedDict): """A single OpenAI-compatible chat message used in accuracy benchmark prompts.""" diff --git a/src/aiperf/dataset/loader/accuracy_dataset_loader.py b/src/aiperf/dataset/loader/accuracy_dataset_loader.py index 5b4b0a5bda..edf737e7ce 100644 --- a/src/aiperf/dataset/loader/accuracy_dataset_loader.py +++ b/src/aiperf/dataset/loader/accuracy_dataset_loader.py @@ -11,9 +11,11 @@ The problem ordering is deterministic: Conversation i corresponds to BenchmarkProblem i. Each Conversation carries accuracy_ground_truth and accuracy_task so that DatasetManager can propagate them through -ConversationMetadata inside DatasetConfiguredNotification. Processors -(AccuracyRecordProcessor, AccuracyResultsProcessor) receive these values -from the notification instead of independently re-loading the benchmark. +ConversationMetadata inside DatasetConfiguredNotification. The ground truth is +read by AccuracyRecordProcessor, which stamps the per-conversation task label +onto each graded record so AccuracyAccumulator can bucket by task. Processors +receive these values from the notification instead of independently re-loading +the benchmark. The session_num % len(conversations) mapping handles both single-pass and multi-pass (num_requests > dataset size) runs and is only valid when the dataset is sampled sequentially; DatasetManager enforces that invariant and diff --git a/src/aiperf/plugin/enums.py b/src/aiperf/plugin/enums.py index 34288af258..6c9952a8ca 100644 --- a/src/aiperf/plugin/enums.py +++ b/src/aiperf/plugin/enums.py @@ -79,7 +79,7 @@ AccumulatorTypeStr: TypeAlias = str AccumulatorType = plugins.create_enum(PluginType.ACCUMULATOR, "AccumulatorType", module=__name__) -"""Dynamic enum for accumulator. Example: AccumulatorType.ACCURACY, AccumulatorType.GPU_TELEMETRY, AccumulatorType.SERVER_METRICS""" +"""Dynamic enum for accumulator. Example: AccumulatorType.ACCURACY, AccumulatorType.METRIC_RESULTS, AccumulatorType.SERVER_METRICS""" StreamExporterTypeStr: TypeAlias = str StreamExporterType = plugins.create_enum(PluginType.STREAM_EXPORTER, "StreamExporterType", module=__name__) diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index 51eda4b82b..48ab29f215 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -1020,14 +1020,6 @@ accumulator: metadata: record_types: [server_metrics] - accuracy_results: - class: aiperf.accuracy.accuracy_results_processor:AccuracyResultsProcessor - description: | - Accuracy accumulator that ingests graded metric records and computes summary - accuracy metrics. Self-disables when accuracy mode is off. - metadata: - record_types: [metric_records] - accuracy: class: aiperf.accuracy.accumulator:AccuracyAccumulator description: | diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 51cebbdd24..15f10be491 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -1102,8 +1102,8 @@ async def _summarize_one_accumulator( Returns the result (or exception object) so a single bad accumulator cannot abort the rest. Accumulators that support phase/window-scoped - export (marked with ``supports_phase_scoped_export`` — MetricsAccumulator - and AccuracyResultsProcessor) get ``export_results(ctx)`` so warmup + export (marked with ``supports_phase_scoped_export`` — MetricsAccumulator) + get ``export_results(ctx)`` so warmup records are excluded from profiling summaries; otherwise prefers ``summarize()`` and falls back to ``export_results(ctx)``. """ diff --git a/tests/unit/accuracy/test_accuracy_record_processor.py b/tests/unit/accuracy/test_accuracy_record_processor.py index 3bfd901bf3..0f1ea2f237 100644 --- a/tests/unit/accuracy/test_accuracy_record_processor.py +++ b/tests/unit/accuracy/test_accuracy_record_processor.py @@ -6,10 +6,8 @@ import pytest from aiperf.accuracy.accuracy_record_processor import AccuracyRecordProcessor -from aiperf.accuracy.accuracy_results_processor import AccuracyResultsProcessor from aiperf.accuracy.models import AccuracyRecordsData, GradingResult from aiperf.common.enums import CreditPhase -from aiperf.common.messages.inference_messages import MetricRecordsData from aiperf.common.models.dataset_models import ConversationMetadata, DatasetMetadata from aiperf.config import BenchmarkRun from aiperf.plugin.enums import ( @@ -46,10 +44,6 @@ def _make_processor(monkeypatch) -> AccuracyRecordProcessor: return AccuracyRecordProcessor(run=_make_run(), service_id="test") -def _make_accuracy_accumulator() -> AccuracyResultsProcessor: - return AccuracyResultsProcessor(run=_make_run()) - - def _make_dataset_metadata( ground_truths: list[str], tasks: list[str] ) -> DatasetMetadata: @@ -68,15 +62,6 @@ def _make_dataset_metadata( ) -def _make_record_data( - session_num: int, correct: float = 1.0, unparsed: float = 0.0 -) -> MetricRecordsData: - return MetricRecordsData( - metadata=create_metric_metadata(session_num=session_num), - metrics={"accuracy_correct": correct, "accuracy_unparsed": unparsed}, - ) - - class TestAccuracyRecordProcessorOnDatasetConfigured: def test_populates_ground_truths_from_metadata(self, monkeypatch) -> None: processor = _make_processor(monkeypatch) @@ -265,123 +250,3 @@ def test_non_verbose_debug_enabled_logs_at_debug(self, monkeypatch) -> None: processor._log_grading_detail(0, "some response", self._result()) assert len(logged) == 1 assert "sandboxed exec failed" in logged[0] - - -class TestAccuracyResultsProcessorOnDatasetConfigured: - def test_populates_tasks_from_metadata(self) -> None: - processor = _make_accuracy_accumulator() - metadata = _make_dataset_metadata(["A", "B"], ["algebra", "history"]) - - processor.on_dataset_configured(metadata) - - assert processor._tasks == ["algebra", "history"] - - def test_skips_conversations_without_accuracy_task(self) -> None: - processor = _make_accuracy_accumulator() - conversations = [ - ConversationMetadata(conversation_id="plain"), - ConversationMetadata( - conversation_id="accurate", - accuracy_ground_truth="B", - accuracy_task="math", - ), - ] - metadata = DatasetMetadata( - conversations=conversations, - sampling_strategy=DatasetSamplingStrategy.SEQUENTIAL, - ) - - processor.on_dataset_configured(metadata) - - assert processor._tasks == ["math"] - - -@pytest.mark.asyncio -class TestAccuracyResultsProcessorSessionBounds: - async def test_process_record_wraps_when_session_num_exceeds_dataset(self) -> None: - """session_num >= dataset size wraps via modulo so the correct task is recorded.""" - processor = _make_accuracy_accumulator() - processor._tasks = ["algebra"] - - # session_num=1 wraps to index 0 (the only task, "algebra") - await processor.process_record(_make_record_data(session_num=1)) - - assert processor._task_total[CreditPhase.PROFILING]["algebra"] == 1 - assert processor._overall_total[CreditPhase.PROFILING] == 1 - - async def test_process_record_wraps_to_correct_task(self) -> None: - """With N problems, session_num=N+1 accumulates under the task at index 1.""" - processor = _make_accuracy_accumulator() - processor._tasks = ["algebra", "history", "biology"] - - # session_num=4 % 3 = index 1 → task="history" - await processor.process_record(_make_record_data(session_num=4)) - - assert processor._task_total[CreditPhase.PROFILING]["history"] == 1 - assert processor._task_total[CreditPhase.PROFILING].get("algebra", 0) == 0 - - async def test_process_record_last_valid_session_num_succeeds(self) -> None: - processor = _make_accuracy_accumulator() - processor._tasks = ["test_task", "test_task"] - - await processor.process_record(_make_record_data(session_num=1, correct=1.0)) - - assert processor._overall_total[CreditPhase.PROFILING] == 1 - assert processor._overall_correct[CreditPhase.PROFILING] == 1 - assert processor._task_correct[CreditPhase.PROFILING]["test_task"] == 1 - - async def test_process_record_raises_if_not_configured(self) -> None: - """process_record must raise if on_dataset_configured was never called.""" - processor = _make_accuracy_accumulator() - - with pytest.raises(RuntimeError, match="dataset not configured"): - await processor.process_record(_make_record_data(session_num=0)) - - async def test_process_record_increments_overall_unparsed(self) -> None: - processor = _make_accuracy_accumulator() - processor._tasks = ["algebra"] - - await processor.process_record( - _make_record_data(session_num=0, correct=1.0, unparsed=1.0) - ) - - assert processor._overall_unparsed[CreditPhase.PROFILING] == 1 - assert processor._overall_total[CreditPhase.PROFILING] == 1 - - async def test_process_record_increments_task_unparsed(self) -> None: - processor = _make_accuracy_accumulator() - processor._tasks = ["algebra"] - - await processor.process_record( - _make_record_data(session_num=0, correct=0.0, unparsed=1.0) - ) - - assert processor._task_unparsed[CreditPhase.PROFILING]["algebra"] == 1 - - async def test_process_record_does_not_increment_unparsed_when_conforming( - self, - ) -> None: - processor = _make_accuracy_accumulator() - processor._tasks = ["algebra"] - - await processor.process_record( - _make_record_data(session_num=0, correct=1.0, unparsed=0.0) - ) - - assert processor._overall_unparsed[CreditPhase.PROFILING] == 0 - assert processor._task_unparsed[CreditPhase.PROFILING].get("algebra", 0) == 0 - - async def test_process_record_missing_unparsed_key_treated_as_conforming( - self, - ) -> None: - """Records without accuracy_unparsed (e.g. from older graders) count as conforming.""" - processor = _make_accuracy_accumulator() - processor._tasks = ["algebra"] - data = MetricRecordsData( - metadata=create_metric_metadata(session_num=0), - metrics={"accuracy_correct": 1.0}, # no accuracy_unparsed key - ) - - await processor.process_record(data) - - assert processor._overall_unparsed[CreditPhase.PROFILING] == 0 From fa08e63e22b314eed41636059c03cf60959102a4 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 20:49:07 -0700 Subject: [PATCH 18/39] fix(accuracy): add numeric bounds to accuracy summary/record fields The finite-invariants ratchet (tests/unit/property/test_finite_invariants.py) requires every Pydantic numeric field to carry a ge/gt/le/lt bound or be FiniteFloat. Add ge=0 to the count/timestamp/index fields and ge=0,le=1 to the rate fields on AccuracyRecordsData/TaskAccuracyStats/AccuracySummary, and baseline the two new messages' inherited request_ns (same pre-existing base debt every message class carries). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/accuracy/models.py | 26 +++++++++++-------- .../property/_numeric_bounds_baseline.txt | 2 ++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index 220903b049..bf9b0813e4 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -72,7 +72,8 @@ class AccuracyRecordsData(AIPerfBaseModel): record_type: ClassVar[str] = "accuracy" session_num: int = Field( - description="Conversation/session index this response came from, used to map to task" + ge=0, + description="Conversation/session index this response came from, used to map to task", ) worker_id: str = Field( description="ID of the record processor that produced this record" @@ -81,7 +82,7 @@ class AccuracyRecordsData(AIPerfBaseModel): description="Benchmark phase active when grading completed (warmup vs profiling)" ) timestamp_ns: int = Field( - description="Nanosecond wall-clock timestamp when grading completed" + ge=0, description="Nanosecond wall-clock timestamp when grading completed" ) task: str | None = Field( default=None, @@ -111,16 +112,16 @@ class AccuracyRecordsData(AIPerfBaseModel): class TaskAccuracyStats(AIPerfBaseModel): """Per-task accuracy rollup.""" - total: int = Field(description="Total responses evaluated for this task") - passed: int = Field(description="Number graded correct for this task") + total: int = Field(ge=0, description="Total responses evaluated for this task") + passed: int = Field(ge=0, description="Number graded correct for this task") unparsed: int = Field( - description="Number that needed a regex fallback for this task" + ge=0, description="Number that needed a regex fallback for this task" ) accuracy_rate: float = Field( - description="passed/total for this task, 0.0 when total==0" + ge=0, le=1, description="passed/total for this task, 0.0 when total==0" ) unparsed_rate: float = Field( - description="unparsed/total for this task, 0.0 when total==0" + ge=0, le=1, description="unparsed/total for this task, 0.0 when total==0" ) @@ -128,16 +129,19 @@ class AccuracySummary(AIPerfBaseModel): """Accumulator result payload; structured replacement for today's list[MetricResult].""" total_evaluated: int = Field( - description="Total responses evaluated across all tasks" + ge=0, description="Total responses evaluated across all tasks" ) total_passed: int = Field( - description="Total responses graded correct across all tasks" + ge=0, description="Total responses graded correct across all tasks" ) accuracy_rate: float = Field( - description="total_passed/total_evaluated, 0.0 when total_evaluated==0" + ge=0, + le=1, + description="total_passed/total_evaluated, 0.0 when total_evaluated==0", ) overall_unparsed: int = Field( - description="Total responses that needed a regex fallback across all tasks" + ge=0, + description="Total responses that needed a regex fallback across all tasks", ) grader_name: str | None = Field( default=None, description="Grader that scored these responses, if uniform" diff --git a/tests/unit/property/_numeric_bounds_baseline.txt b/tests/unit/property/_numeric_bounds_baseline.txt index 81e4a07680..69eff8d5f9 100644 --- a/tests/unit/property/_numeric_bounds_baseline.txt +++ b/tests/unit/property/_numeric_bounds_baseline.txt @@ -39,6 +39,7 @@ AioHttpTraceData.response_receive_start_perf_ns: numeric field has no ge/gt/le/l AioHttpTraceData.response_status_code: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. AioHttpTraceData.tcp_connect_end_perf_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. AioHttpTraceData.tcp_connect_start_perf_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. +AccuracyRecordsMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. AllRecordsReceivedMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. AnalysisStats.total_requests: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. AnalysisStats.unique_prefixes: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. @@ -273,6 +274,7 @@ ProcessingStats.errors: numeric field has no ge/gt/le/lt bound and is not Finite ProcessingStats.processed: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. ProcessRecordsCommand.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. ProcessRecordsResponse.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. +ProcessAccuracyResultMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. ProcessRecordsResultMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. ProcessServerMetricsResultMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. ProcessTelemetryResultMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. From e381e96287f1febd22b3e73ee5caf174c17e41f9 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 20:53:16 -0700 Subject: [PATCH 19/39] docs(accuracy): document dedicated accuracy record-type channel Accuracy benchmarking now rides a first-class 'accuracy' record-type channel (like gpu_telemetry/server_metrics) instead of fake metric keys on the shared metric_records channel. Update architecture.md with a Record-Type Channels section, refresh the accuracy-benchmarking Output/ Architecture sections (Unparsed column, accuracy_export.jsonl), and correct accuracy_stubs.md to describe AccuracyAccumulator/AccuracySummary/ AccuracyJSONLWriter in place of the removed AccuracyResultsProcessor. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/accuracy/accuracy-benchmarking.md | 53 ++++++++---- docs/accuracy/accuracy_stubs.md | 107 ++++++++++++++++++++----- docs/architecture.md | 29 +++++++ 3 files changed, 153 insertions(+), 36 deletions(-) diff --git a/docs/accuracy/accuracy-benchmarking.md b/docs/accuracy/accuracy-benchmarking.md index 6fe742ec9e..821216ab21 100644 --- a/docs/accuracy/accuracy-benchmarking.md +++ b/docs/accuracy/accuracy-benchmarking.md @@ -332,29 +332,50 @@ When extraction fell back past the `\boxed{}` step (i.e. the model didn't follow ## Output -Accuracy results are displayed in the console and exported to CSV: +Accuracy flows on a dedicated `accuracy` record-type channel (alongside the +`metric_records`, `gpu_telemetry`, and `server_metrics` channels — see +[Record-Type Channels](../architecture.md#record-type-channels)). Each graded +response is routed to two sinks: an accumulator that produces the per-task +summary, and a per-record JSONL writer. + +Accuracy results are displayed in the console and exported to CSV. The console +table and the CSV both carry a per-task `Unparsed` count (responses where the +grader needed a regex fallback because the model output did not match the +expected format): ```text - Accuracy Benchmark Results -┏━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━┓ -┃ Task ┃ Correct ┃ Total ┃ Accuracy ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━┩ -│ abstract_algebra │ 35 │ 100 │ 35.00% │ -│ ... │ ... │ ... │ ... │ -│ OVERALL │ 8368 │ 14042 │ 59.59% │ -└─────────────────────────┴─────────┴───────┴──────────┘ + Accuracy Benchmark Results +┏━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━┓ +┃ Task ┃ Correct ┃ Total ┃ Unparsed ┃ Accuracy ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━┩ +│ abstract_algebra │ 35 │ 100 │ 2 │ 35.00% │ +│ ... │ ... │ ... │ ... │ ... │ +│ OVERALL │ 8368 │ 14042 │ 61 │ 59.59% │ +└─────────────────────────┴─────────┴───────┴──────────┴──────────┘ ``` -CSV file: `/accuracy_results.csv` +**Summary CSV:** `/accuracy_results.csv` — one row per task plus a +trailing `OVERALL` row. Columns: `task, total, passed, unparsed, accuracy_rate, +unparsed_rate`. + +**Per-record JSONL:** `/accuracy_export.jsonl` (or +`_accuracy.jsonl` when an artifact prefix is configured) — one JSON line +per graded response with the full grading detail that the summary rolls up. +Each line carries: `session_num`, `worker_id`, `benchmark_phase`, +`timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, `confidence`, +`expected` (ground truth), `actual` (extracted answer), and `reasoning`. Use it +for per-response post-hoc analysis — e.g. finding which prompts were graded +`unparsed` and why. ## Architecture -```text -AccuracyDatasetLoader → Conversation/Turn objects (dataset pipeline) -AccuracyRecordProcessor → grades each response (record pipeline) -AccuracyResultsProcessor → aggregates per-task accuracy (results pipeline) -AccuracyConsoleExporter → Rich table output -AccuracyDataExporter → CSV export +```mermaid +flowchart LR + DL[AccuracyDatasetLoader] -->|Conversation/Turn objects| RP[AccuracyRecordProcessor
grades each response] + RP -->|AccuracyRecordsData
on the accuracy channel| ACC[AccuracyAccumulator
per-task AccuracySummary] + RP -->|AccuracyRecordsData| JW[AccuracyJSONLWriter
accuracy_export.jsonl] + ACC --> CE[AccuracyConsoleExporter
Rich table] + ACC --> DE[AccuracyDataExporter
accuracy_results.csv] ``` All components self-disable when `--accuracy-benchmark` is not set. diff --git a/docs/accuracy/accuracy_stubs.md b/docs/accuracy/accuracy_stubs.md index b654e99433..1c8ae952ca 100644 --- a/docs/accuracy/accuracy_stubs.md +++ b/docs/accuracy/accuracy_stubs.md @@ -7,6 +7,8 @@ This document catalogs every stubbed method in the accuracy benchmarking scaffolding. The scaffolding is fully integrated into the plugin system, CLI, and config pipeline — the performance benchmarking path is unaffected. +> **Pipeline note:** Accuracy is now a first-class **dedicated `accuracy` record-type channel** (like `gpu_telemetry` and `server_metrics`), not a shoehorn on the shared `metric_records` channel. Graded responses flow as typed `AccuracyRecordsData` and are rolled up by `AccuracyAccumulator` into a structured `AccuracySummary`; a per-record `AccuracyJSONLWriter` streams the full grading detail. The old `AccuracyResultsProcessor` (which returned `list[MetricResult]` on the `metric_records` channel via `accuracy.` tags) has been removed. See [Record-Type Channels](../architecture.md#record-type-channels) for the current data flow. The registrations below have been updated accordingly. + **Status summary:** All accuracy scaffolding is now implemented end-to-end. The eight graders (`MultipleChoiceGrader`, `MathGrader`, `CodeExecutionGrader`, `LightevalExprGrader`, `LightevalLatexGrader`, `LightevalGPQAGrader`, `LightevalGSM8KGrader`, `ExactMatchGrader`) and ten benchmark loaders (`MMLUBenchmark`, `AIMEBenchmark`, `HellaSwagBenchmark`, `BigBenchBenchmark`, `AIME24Benchmark`, `AIME25Benchmark`, `Math500Benchmark`, `GPQADiamondBenchmark`, `LCBCodeGenerationBenchmark`, `GSM8KBenchmark`) are all wired into the plugin system, CLI, config pipeline, and processor/exporter chain. There are no remaining stubs. ## Table of Contents @@ -32,10 +34,15 @@ graph TD A --> C[AccuracyGrader
4 graders
grade + extract] B --> D[AccuracyRecordProcessor
process_record] C --> D - D --> E[AccuracyResultsProcessor
process_result
summarize] + D --> E[AccuracyAccumulator
export_results
AccuracySummary] + D --> J[AccuracyJSONLWriter
accuracy_export.jsonl] E --> F[AccuracyConsoleExporter
AccuracyDataExporter] ``` +`AccuracyRecordProcessor.process_record` returns a typed `AccuracyRecordsData` +that flows on the dedicated `accuracy` channel; the Records Manager fans it out +to `AccuracyAccumulator` and `AccuracyJSONLWriter`. + All processors and exporters **self-disable** when `cfg.accuracy.enabled is False` by raising their respective `Disabled` exceptions in `__init__`. This is the same pattern used by `RawRecordWriterProcessor`, `ServerMetricsCsvExporter`, etc. --- @@ -71,6 +78,45 @@ class BenchmarkProblem(AIPerfBaseModel): raw_messages: list[dict] | None = None # Preformatted messages, when supplied by a loader ``` +### AccuracyRecordsData + +Typed per-graded-response record carried on the dedicated `accuracy` channel. +`record_type` is a plain `ClassVar` (`"accuracy"`), mirroring `ServerMetricsRecord` +and `TelemetryRecord`. + +```python +class AccuracyRecordsData(AIPerfBaseModel): + record_type: ClassVar[str] = "accuracy" + session_num: int # session/conversation index + worker_id: str # record processor that produced this record + benchmark_phase: CreditPhase # warmup vs profiling + timestamp_ns: int # wall-clock ns when grading completed + task: str | None # accuracy task/subtask (None when unlabeled) + grader_name: str # which grader scored this response + passed: bool # graded correct (from GradingResult.correct) + unparsed: bool = False # needed a regex fallback (from GradingResult.unparsed) + confidence: float # grading confidence 0.0-1.0 + expected: str # ground truth (from GradingResult.ground_truth) + actual: str # extracted answer (from GradingResult.extracted_answer) + reasoning: str # grader's explanation +``` + +### AccuracySummary + +Structured accumulator result (replaces the old `list[MetricResult]`). +`AccuracyAccumulator` builds it; the console/CSV exporters render from it. +`to_csv()` emits per-task rows plus a trailing `OVERALL` row. + +```python +class AccuracySummary(AIPerfBaseModel): + total_evaluated: int + total_passed: int + accuracy_rate: float + overall_unparsed: int + grader_name: str | None + per_task: dict[str, TaskAccuracyStats] # total/passed/unparsed/accuracy_rate/unparsed_rate +``` + --- ## Protocols @@ -205,27 +251,45 @@ This class is fully implemented and serves as the canonical reference for wiring ```python async def process_record( self, record: ParsedResponseRecord, metadata: MetricRecordMetadata -) -> MetricRecordDict # IMPLEMENTED in PR #815 +) -> AccuracyRecordsData # typed record on the accuracy channel ``` **Reference implementation:** `MetricRecordProcessor` in `src/aiperf/post_processors/metric_record_processor.py` -### AccuracyResultsProcessor — IMPLEMENTED in PR #815 +### AccuracyAccumulator — accuracy channel -**File:** `src/aiperf/accuracy/accuracy_results_processor.py` -**Parent:** `AIPerfLifecycleMixin` -**Implements:** `AccumulatorProtocol` -**Plugin key:** `accuracy_results` (under `accumulator`) +**File:** `src/aiperf/accuracy/accumulator.py` +**Parent:** `BaseMetricsProcessor` +**Plugin key:** `accuracy` (under `accumulator`, `record_types: [accuracy]`) **Disables via:** `PostProcessorDisabled` when `not cfg.accuracy.enabled` -This class is fully implemented and serves as the canonical reference for aggregating per-task accuracy metrics from routed `metric_records`. +Ingests per-graded-response `AccuracyRecordsData` and rolls them up into a +structured `AccuracySummary` (overall + per-task pass rates and unparsed +counts). Phase-scoped export mirrors `ServerMetricsAccumulator`. ```python -async def process_record(self, record_data: MetricRecordsData) -> None # IMPLEMENTED in PR #815 -async def summarize(self) -> list[MetricResult] # IMPLEMENTED in PR #815 +async def process_record(self, record: AccuracyRecordsData) -> None +async def export_results(self, ctx: ExportContext) -> AccuracySummary | None # phase-scoped +async def summarize(self, ctx: SummaryContext | None = None) -> AccuracySummary | None ``` -**Reference implementation:** `MetricsAccumulator` in `src/aiperf/metrics/accumulator.py` +**Reference implementation:** `ServerMetricsAccumulator` in `src/aiperf/server_metrics/accumulator.py` + +### AccuracyJSONLWriter — accuracy channel (per-record stream) + +**File:** `src/aiperf/accuracy/jsonl_writer.py` +**Parent:** `BaseMetricsProcessor`, `BufferedJSONLWriterMixin[AccuracyRecordsData]` +**Plugin key:** `accuracy_jsonl_writer` (under `stream_exporter`, `record_types: [accuracy]`) +**Disables via:** `PostProcessorDisabled` when `not cfg.accuracy.enabled` + +Streams each `AccuracyRecordsData` to `/accuracy_export.jsonl` +(one JSON line per graded response) — the per-record grading detail that was +previously discarded. + +```python +async def process_record(self, record: AccuracyRecordsData) -> None +async def finalize(self) -> None +``` --- @@ -284,12 +348,13 @@ All stubs are registered in `src/aiperf/plugin/plugins.yaml` and `src/aiperf/plu ### Registrations in Existing Categories -| Category | Plugin Key | Class | -|----------|-----------|-------| -| `record_processor` | `accuracy_record` | `AccuracyRecordProcessor` | -| `accumulator` | `accuracy_results` | `AccuracyResultsProcessor` | -| `console_exporter` | `accuracy` | `AccuracyConsoleExporter` | -| `data_exporter` | `accuracy_csv` | `AccuracyDataExporter` | +| Category | Plugin Key | Class | Metadata | +|----------|-----------|-------|----------| +| `record_processor` | `accuracy_record` | `AccuracyRecordProcessor` | — | +| `accumulator` | `accuracy` | `AccuracyAccumulator` | `record_types: [accuracy]` | +| `stream_exporter` | `accuracy_jsonl_writer` | `AccuracyJSONLWriter` | `record_types: [accuracy]` | +| `console_exporter` | `accuracy` | `AccuracyConsoleExporter` | — | +| `data_exporter` | `accuracy_csv` | `AccuracyDataExporter` | — | --- @@ -302,11 +367,12 @@ All stubs are registered in `src/aiperf/plugin/plugins.yaml` and `src/aiperf/plu | Graders | 7 (all) | 0 | — | 0 | | Benchmarks | 9 (all) | 0 | — | 0 | | Record Processor | 1 (`AccuracyRecordProcessor`) | 0 | — | 0 | -| Accuracy Accumulator | 1 (`AccuracyResultsProcessor`) | 0 | — | 0 | +| Accuracy Accumulator | 1 (`AccuracyAccumulator`) | 0 | — | 0 | +| Accuracy JSONL Writer | 1 (`AccuracyJSONLWriter`) | 0 | — | 0 | | Console Exporter | 1 (`AccuracyConsoleExporter`) | 0 | — | 0 | | Data Exporter | 1 (`AccuracyDataExporter`) | 0 | — | 0 | | Stub-plugin Validator | 1 (`AccuracyConfig._reject_stub_plugins`, idle until next stub) | 0 | — | 0 | -| **Total** | **21** | **0** | | **0** | +| **Total** | **22** | **0** | | **0** | ### Self-Disabling Pattern @@ -323,7 +389,8 @@ The processors, exporters, all seven graders, and all nine benchmarks are wired | **Canonical grader** | `src/aiperf/accuracy/graders/multiple_choice.py` | | **Canonical benchmark** | `src/aiperf/accuracy/benchmarks/mmlu.py` | | **Canonical record processor** | `src/aiperf/accuracy/accuracy_record_processor.py` | -| **Canonical accuracy accumulator** | `src/aiperf/accuracy/accuracy_results_processor.py` | +| **Canonical accuracy accumulator** | `src/aiperf/accuracy/accumulator.py` | +| **Accuracy per-record JSONL writer** | `src/aiperf/accuracy/jsonl_writer.py` | | **Canonical console exporter** | `src/aiperf/accuracy/accuracy_console_exporter.py` | | **Canonical data exporter** | `src/aiperf/accuracy/accuracy_data_exporter.py` | | Disabled exception pattern | `src/aiperf/post_processors/raw_record_writer_processor.py:47` | diff --git a/docs/architecture.md b/docs/architecture.md index 8cf46fe8bb..651f534cc1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -114,6 +114,35 @@ The Records Manager handles the collection, organization, and storage of benchma - Supporting the generation of reports and artifacts for performance evaluation - Managing the final export of aggregated performance summaries and per-request details +#### Record-Type Channels + +The Records Manager does not hard-wire one handler per producer. Instead, every typed record carries a `record_type` class attribute and is fanned out through a metadata-driven routing table: each accumulator and stream exporter declares the record types it consumes via `record_types` in `plugins.yaml`, and `_dispatch_record` routes each record by `getattr(record, "record_type")` to all registered handlers. This makes each record type a dedicated channel. + +Accuracy benchmarking is one such dedicated channel, joining `metric_records`, `gpu_telemetry`, and `server_metrics`: + +| Channel (`record_type`) | Producer | Message | Consumers (`record_types`) | +|---|---|---|---| +| `metric_records` | Record Processor | `MetricRecordsMessage` | `MetricsAccumulator`, JSONL/OTel stream exporters | +| `gpu_telemetry` | GPU Telemetry Manager | `TelemetryRecordsMessage` | `GPUTelemetryAccumulator`, GPU JSONL writer | +| `server_metrics` | Server Metrics Manager | `ServerMetricsRecordMessage` | `ServerMetricsAccumulator`, server-metrics JSONL writer | +| `accuracy` | Record Processor (`AccuracyRecordProcessor`) | `AccuracyRecordsMessage` | `AccuracyAccumulator`, `AccuracyJSONLWriter` | + +The `AccuracyRecordProcessor` grades each parsed response against ground truth and returns an `AccuracyRecordsData` (`session_num`, `worker_id`, `benchmark_phase`, `timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, `confidence`, `expected`, `actual`, `reasoning`). The `RecordProcessorService` partitions these typed records off the metric-dict stream and pushes an `AccuracyRecordsMessage` to the Records Manager, which fans each record out to the `AccuracyAccumulator` (rolls records into an `AccuracySummary` of overall + per-task pass/unparsed rates) and the `AccuracyJSONLWriter` (streams the full per-response grading detail to `accuracy_export.jsonl`). + +At end of the profiling phase, `RecordsManager._publish_accuracy_results(phase)` exports the phase-scoped `AccuracySummary` and publishes a `ProcessAccuracyResultMessage`. The `SystemController` receives it, stores the summary, and hands it to the exporters via `ExporterConfig.accuracy_results` — gated in shutdown like the telemetry and server-metrics results. The `AccuracyConsoleExporter` and `AccuracyDataExporter` render from that structured summary rather than filtering tagged metric results. + +```mermaid +flowchart TD + W[Worker] -->|raw response| RP[AccuracyRecordProcessor
grade vs ground truth] + RP -->|AccuracyRecordsData
record_type=accuracy| RM[RecordsManager
routing table] + RM -->|process_record| ACC[AccuracyAccumulator
AccuracySummary] + RM -->|process_record| JW[AccuracyJSONLWriter
accuracy_export.jsonl] + ACC -->|export_results phase=PROFILING| PUB[_publish_accuracy_results] + PUB -->|ProcessAccuracyResultMessage| SC[SystemController
stores AccuracySummary] + SC -->|ExporterConfig.accuracy_results| CE[AccuracyConsoleExporter] + SC -->|ExporterConfig.accuracy_results| DE[AccuracyDataExporter
accuracy_results.csv] +``` + ### GPU Telemetry Manager The GPU Telemetry Manager collects GPU metrics during benchmarking runs via pluggable collectors. From 52b9d9e42f66d08102a047f088ce8126cee7dc51 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 21:12:11 -0700 Subject: [PATCH 20/39] fix(accuracy): reproduce byte-identical legacy accuracy exports from the dedicated channel Convert the dedicated-channel AccuracySummary into the legacy accuracy.* MetricResults (AccuracySummary.to_metric_results) and inject them into ProfileResults.records at the SystemController export site, guarded against double-append. Restore the legacy tag constants/helpers and the record-reading accuracy CSV/console exporters so every legacy exporter reproduces byte-identical output. The dedicated channel internals (record type, accumulator, jsonl writer, result message, wait-gating) are unchanged; only the OUTPUT representation is materialized from the summary. accuracy_export.jsonl is kept (additive). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- .../accuracy/accuracy_console_exporter.py | 88 +++++++--- src/aiperf/accuracy/accuracy_data_exporter.py | 72 ++++++-- src/aiperf/accuracy/models.py | 95 ++++++++++- src/aiperf/controller/system_controller.py | 22 +++ .../test_accuracy_console_exporter.py | 157 ++++++++++-------- .../accuracy/test_accuracy_data_exporter.py | 114 +++++++------ .../test_summary_to_metric_results.py | 91 ++++++++++ .../controller/test_accuracy_shutdown_gate.py | 47 ++++++ 8 files changed, 525 insertions(+), 161 deletions(-) create mode 100644 tests/unit/accuracy/test_summary_to_metric_results.py diff --git a/src/aiperf/accuracy/accuracy_console_exporter.py b/src/aiperf/accuracy/accuracy_console_exporter.py index 435f55aee6..1a5458ead5 100644 --- a/src/aiperf/accuracy/accuracy_console_exporter.py +++ b/src/aiperf/accuracy/accuracy_console_exporter.py @@ -5,6 +5,13 @@ from typing import TYPE_CHECKING, Any +from aiperf.accuracy.models import ( + ACCURACY_METRIC_PREFIX, + ACCURACY_OVERALL_TAG, + ACCURACY_TASK_TAG_PREFIX, + ACCURACY_UNPARSED_TAG, + ACCURACY_UNPARSED_TASK_TAG_PREFIX, +) from aiperf.common.exceptions import ConsoleExporterDisabled from aiperf.common.mixins import AIPerfLoggerMixin from aiperf.exporters.exporter_config import ExporterConfig @@ -12,15 +19,11 @@ if TYPE_CHECKING: from rich.console import Console - from aiperf.accuracy.models import AccuracySummary - class AccuracyConsoleExporter(AIPerfLoggerMixin): """Console exporter for accuracy benchmarking results. - Renders a Rich table with per-task accuracy breakdown and overall score, - sourced from the structured ``AccuracySummary`` delivered on the dedicated - accuracy channel. + Renders a Rich table with per-task accuracy breakdown and overall score. """ def __init__(self, exporter_config: ExporterConfig, **kwargs: Any) -> None: @@ -36,15 +39,37 @@ def __init__(self, exporter_config: ExporterConfig, **kwargs: Any) -> None: async def export(self, console: Console) -> None: """Render accuracy results as a Rich table to the given console. - Prints a per-task breakdown (passed / total / accuracy%) followed by an - OVERALL row. Does nothing when no accuracy summary was delivered. + Prints a per-task breakdown (correct / total / accuracy%) followed by an + OVERALL row. Does nothing if no ``accuracy.*`` metrics are present in + ``exporter_config.results``. """ from rich.table import Table - summary = self.exporter_config.accuracy_results - if summary is None: + results = self.exporter_config.results + if results is None or results.records is None: + return + + accuracy_metrics = [ + r for r in results.records if r.tag.startswith(ACCURACY_METRIC_PREFIX) + ] + if not accuracy_metrics: return + overall = next( + (m for m in accuracy_metrics if m.tag == ACCURACY_OVERALL_TAG), None + ) + task_metrics = [ + m for m in accuracy_metrics if m.tag.startswith(ACCURACY_TASK_TAG_PREFIX) + ] + unparsed_overall = next( + (m for m in accuracy_metrics if m.tag == ACCURACY_UNPARSED_TAG), None + ) + unparsed_by_task: dict[str, int] = { + m.tag.removeprefix(ACCURACY_UNPARSED_TASK_TAG_PREFIX): int(m.sum or 0) + for m in accuracy_metrics + if m.tag.startswith(ACCURACY_UNPARSED_TASK_TAG_PREFIX) + } + table = Table(title="Accuracy Benchmark Results", show_lines=True) table.add_column("Task", style="cyan", min_width=30) table.add_column("Correct", justify="right") @@ -52,44 +77,57 @@ async def export(self, console: Console) -> None: table.add_column("Unparsed", justify="right", style="yellow") table.add_column("Accuracy", justify="right", style="bold") - for task_name, stats in sorted(summary.per_task.items()): + for m in task_metrics: + task_name = m.tag.removeprefix(ACCURACY_TASK_TAG_PREFIX) + acc_str = f"{m.current:.2%}" if m.current is not None else "N/A" + unparsed_count = str(unparsed_by_task.get(task_name, 0)) table.add_row( task_name, - str(stats.passed), - str(stats.total), - str(stats.unparsed), - f"{stats.accuracy_rate:.2%}", + str(m.sum or 0), + str(m.count or 0), + unparsed_count, + acc_str, ) - if summary.total_evaluated: + if overall: + acc_str = f"{overall.current:.2%}" if overall.current is not None else "N/A" + overall_unparsed = str( + int(unparsed_overall.sum or 0) if unparsed_overall else 0 + ) table.add_row( "[bold]OVERALL[/bold]", - str(summary.total_passed), - str(summary.total_evaluated), - str(summary.overall_unparsed), - f"[bold green]{summary.accuracy_rate:.2%}[/bold green]", + str(overall.sum or 0), + str(overall.count or 0), + overall_unparsed, + f"[bold green]{acc_str}[/bold green]", style="on dark_green", ) console.print() console.print(table) - self._maybe_warn_all_unparsed(console, summary) + self._maybe_warn_all_unparsed(console, overall, unparsed_overall) def _maybe_warn_all_unparsed( self, console: Console, - summary: AccuracySummary, + overall: Any, + unparsed_overall: Any, ) -> None: """Loud-but-actionable diagnostic for the "accuracy=0 because the server, not the model" case. - Triggers when every evaluated response reports unparsed output — almost + Triggers when every task reports 100% unparsed responses — almost always a mock server or misconfigured endpoint, not an accuracy - problem. Does not gate on total count so it fires on tiny smoke runs. + problem. Does not gate on overall_total so it fires on tiny smoke + runs. """ if not ( - summary.total_evaluated - and summary.overall_unparsed >= summary.total_evaluated + overall + and overall.count + and unparsed_overall + and unparsed_overall.sum is not None + and unparsed_overall.count + and int(unparsed_overall.sum) >= int(unparsed_overall.count) ): return # Console-only diagnostic: export() legitimately runs once per target diff --git a/src/aiperf/accuracy/accuracy_data_exporter.py b/src/aiperf/accuracy/accuracy_data_exporter.py index 8f6613a37b..e291936013 100644 --- a/src/aiperf/accuracy/accuracy_data_exporter.py +++ b/src/aiperf/accuracy/accuracy_data_exporter.py @@ -8,19 +8,26 @@ from pathlib import Path from typing import Any +from aiperf.accuracy.models import ( + ACCURACY_METRIC_PREFIX, + ACCURACY_OVERALL_TAG, + ACCURACY_TASK_TAG_PREFIX, + ACCURACY_UNPARSED_TAG, + ACCURACY_UNPARSED_TASK_TAG_PREFIX, +) from aiperf.common.exceptions import DataExporterDisabled from aiperf.common.mixins import AIPerfLoggerMixin from aiperf.exporters.exporter_config import ExporterConfig, FileExportInfo -_CSV_COLUMNS = ["task", "total", "passed", "unparsed", "accuracy_rate", "unparsed_rate"] +AccuracyCsvRow = tuple[ + str, int, int, int, str +] # (task, correct, total, unparsed, accuracy) class AccuracyDataExporter(AIPerfLoggerMixin): """Data exporter for accuracy benchmarking results. - Exports the per-task accuracy summary (plus an OVERALL row) to CSV for - offline analysis, sourced from the structured ``AccuracySummary`` delivered - on the dedicated accuracy channel. + Exports per-task accuracy summary to CSV for offline analysis. """ def __init__(self, exporter_config: ExporterConfig, **kwargs: Any) -> None: @@ -44,23 +51,58 @@ def get_export_info(self) -> FileExportInfo: ) async def export(self) -> None: - """Write the per-task accuracy summary to CSV at the path from ``get_export_info``. + """Write per-task accuracy summary to CSV at the path from ``get_export_info``. - Columns: task, total, passed, unparsed, accuracy_rate, unparsed_rate. - One row per task plus a final OVERALL row. Does nothing when no accuracy - summary was delivered. + Columns: task, correct, total, accuracy (4 decimal places). Rows are + emitted for each ``accuracy.task.*`` metric plus a final OVERALL row. + Does nothing if no ``accuracy.*`` metrics are present in results. """ - summary = self.exporter_config.accuracy_results - if summary is None: + results = self.exporter_config.results + if results is None or results.records is None: return - rows = summary.to_csv() + accuracy_metrics = [ + r for r in results.records if r.tag.startswith(ACCURACY_METRIC_PREFIX) + ] + if not accuracy_metrics: + return + + unparsed_overall = next( + (m for m in accuracy_metrics if m.tag == ACCURACY_UNPARSED_TAG), None + ) + unparsed_by_task: dict[str, int] = { + m.tag.removeprefix(ACCURACY_UNPARSED_TASK_TAG_PREFIX): int(m.sum or 0) + for m in accuracy_metrics + if m.tag.startswith(ACCURACY_UNPARSED_TASK_TAG_PREFIX) + } + + rows: list[AccuracyCsvRow] = [] + for m in accuracy_metrics: + if m.tag == ACCURACY_OVERALL_TAG: + task_name = "OVERALL" + unparsed = int(unparsed_overall.sum or 0) if unparsed_overall else 0 + elif m.tag.startswith(ACCURACY_TASK_TAG_PREFIX): + task_name = m.tag.removeprefix(ACCURACY_TASK_TAG_PREFIX) + unparsed = unparsed_by_task.get(task_name, 0) + else: + continue + rows.append( + ( + task_name, + int(m.sum or 0), + int(m.count or 0), + unparsed, + f"{m.current:.4f}" if m.current is not None else "", + ) + ) + await asyncio.to_thread(self._write_csv, rows) self.info(f"Accuracy results exported to {self._csv_path}") - def _write_csv(self, rows: list[dict[str, Any]]) -> None: + def _write_csv(self, rows: list[AccuracyCsvRow]) -> None: self._csv_path.parent.mkdir(parents=True, exist_ok=True) with open(self._csv_path, "w", newline="") as f: - writer = csv.DictWriter(f, fieldnames=_CSV_COLUMNS) - writer.writeheader() - writer.writerows(rows) + writer = csv.writer(f) + writer.writerow(["task", "correct", "total", "unparsed", "accuracy"]) + for row in rows: + writer.writerow(row) diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index bf9b0813e4..c305dab530 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -3,7 +3,7 @@ from __future__ import annotations -from typing import Any, ClassVar, Literal +from typing import TYPE_CHECKING, Any, ClassVar, Literal from pydantic import Field from typing_extensions import TypedDict @@ -11,6 +11,27 @@ from aiperf.common.enums import CreditPhase from aiperf.common.models.base_models import AIPerfBaseModel +if TYPE_CHECKING: + from aiperf.common.models import MetricResult + +# Summary/metric tags (the ``accuracy.`` dot namespace) materialized from the +# dedicated-channel ``AccuracySummary`` and read by the accuracy exporters. +ACCURACY_OVERALL_TAG = "accuracy.overall" +ACCURACY_TASK_TAG_PREFIX = "accuracy.task." +ACCURACY_UNPARSED_TAG = "accuracy.unparsed" +ACCURACY_UNPARSED_TASK_TAG_PREFIX = "accuracy.unparsed.task." +ACCURACY_METRIC_PREFIX = "accuracy." + + +def accuracy_task_tag(task: str) -> str: + """Build the MetricResult.tag for a per-task accuracy result.""" + return f"{ACCURACY_TASK_TAG_PREFIX}{task}" + + +def accuracy_unparsed_task_tag(task: str) -> str: + """Build the MetricResult.tag for a per-task unparsed-count result.""" + return f"{ACCURACY_UNPARSED_TASK_TAG_PREFIX}{task}" + class AccuracyChatMessage(TypedDict): """A single OpenAI-compatible chat message used in accuracy benchmark prompts.""" @@ -188,6 +209,78 @@ def to_csv(self) -> list[dict[str, Any]]: ) return rows + def to_metric_results(self) -> list[MetricResult]: + """Legacy ``accuracy.*`` MetricResult representation for byte-identical export. + + Reproduces the legacy ``AccuracyResultsProcessor._build_results`` output + field-for-field so every legacy exporter (perf CSV/JSON + the dedicated + accuracy CSV/console) renders identical bytes when these results are + injected into ``ProfileResults.records``. + + Emitted in this exact order (load-bearing for byte-exact JSON/CSV): + overall, tasks sorted, unparsed overall, unparsed tasks sorted. + """ + from aiperf.common.enums import MetricConsoleGroup + from aiperf.common.models import MetricResult + + results: list[MetricResult] = [] + + if self.total_evaluated > 0: + results.append( + MetricResult( + tag=ACCURACY_OVERALL_TAG, + header="Accuracy (Overall)", + unit="ratio", + count=self.total_evaluated, + current=self.total_passed / self.total_evaluated, + sum=self.total_passed, + console_group=MetricConsoleGroup.NONE, + ) + ) + + for task in sorted(self.per_task): + stats = self.per_task[task] + results.append( + MetricResult( + tag=accuracy_task_tag(task), + header=f"Accuracy ({task})", + unit="ratio", + count=stats.total, + current=stats.passed / stats.total if stats.total else 0.0, + sum=stats.passed, + console_group=MetricConsoleGroup.NONE, + ) + ) + + if self.total_evaluated > 0: + results.append( + MetricResult( + tag=ACCURACY_UNPARSED_TAG, + header="Accuracy Unparsed (Overall)", + unit="ratio", + count=self.total_evaluated, + current=self.overall_unparsed / self.total_evaluated, + sum=self.overall_unparsed, + console_group=MetricConsoleGroup.NONE, + ) + ) + + for task in sorted(self.per_task): + stats = self.per_task[task] + results.append( + MetricResult( + tag=accuracy_unparsed_task_tag(task), + header=f"Accuracy Unparsed ({task})", + unit="ratio", + count=stats.total, + current=stats.unparsed / stats.total if stats.total else 0.0, + sum=stats.unparsed, + console_group=MetricConsoleGroup.NONE, + ) + ) + + return results + class ProcessAccuracyResult(AIPerfBaseModel): """Wire wrapper for a processed accuracy summary - mirrors ProcessServerMetricsResult.""" diff --git a/src/aiperf/controller/system_controller.py b/src/aiperf/controller/system_controller.py index 28dba50814..c686e812ba 100644 --- a/src/aiperf/controller/system_controller.py +++ b/src/aiperf/controller/system_controller.py @@ -162,6 +162,7 @@ def __init__( self._telemetry_results: TelemetryExportData | None = None self._server_metrics_results: ServerMetricsResults | None = None self._accuracy_results: AccuracySummary | None = None + self._accuracy_results_injected = False self._profile_results_received = False self._should_wait_for_telemetry = False self._should_wait_for_server_metrics = False @@ -1068,6 +1069,25 @@ def _print_exit_errors_and_log_file(self) -> None: console.print() console.file.flush() + def _inject_accuracy_results_into_records(self) -> None: + """Materialize the dedicated-channel accuracy summary into the profile records. + + The accuracy computation/transport lives on its own dedicated channel + (``AccuracyAccumulator`` -> ``AccuracySummary``), but legacy exporters + (perf CSV/JSON + the accuracy CSV/console) read ``accuracy.*`` MetricResults + from ``ProfileResults.records``. Convert the summary to those MetricResults + and append them at the END (so JSON key order matches legacy: accuracy.* + after all perf metrics). Guarded so a re-export cannot double-append. + """ + if self._accuracy_results is None or self._accuracy_results_injected: + return + if not self._profile_results or self._profile_results.results.records is None: + return + self._profile_results.results.records.extend( + self._accuracy_results.to_metric_results() + ) + self._accuracy_results_injected = True + async def _print_post_benchmark_info_and_metrics(self) -> None: """Print post benchmark info and metrics to the console.""" if not self._profile_results or not self._profile_results.results.records: @@ -1118,6 +1138,8 @@ async def _print_post_benchmark_info_and_metrics(self) -> None: if console.width < 100: console.width = 100 + self._inject_accuracy_results_into_records() + exporter_manager = ExporterManager( results=self._profile_results.results, run=self.run, diff --git a/tests/unit/accuracy/test_accuracy_console_exporter.py b/tests/unit/accuracy/test_accuracy_console_exporter.py index e4cbe8856c..820613b13d 100644 --- a/tests/unit/accuracy/test_accuracy_console_exporter.py +++ b/tests/unit/accuracy/test_accuracy_console_exporter.py @@ -8,53 +8,61 @@ from rich.console import Console from aiperf.accuracy.accuracy_console_exporter import AccuracyConsoleExporter -from aiperf.accuracy.models import AccuracySummary, TaskAccuracyStats -from aiperf.common.exceptions import ConsoleExporterDisabled +from aiperf.common.models import MetricResult +from aiperf.common.models.record_models import ProfileResults from aiperf.exporters.exporter_config import ExporterConfig from aiperf.plugin.enums import AccuracyBenchmarkType, EndpointType from tests.unit.conftest import make_benchmark_run -def _make_exporter(summary: AccuracySummary | None) -> AccuracyConsoleExporter: +def _make_exporter(records: list[MetricResult] | None) -> AccuracyConsoleExporter: cfg = make_benchmark_run( model_names=["test-model"], endpoint_type=EndpointType.COMPLETIONS, streaming=False, accuracy={"benchmark": AccuracyBenchmarkType.MMLU}, ).cfg + results = ( + ProfileResults(records=records, completed=0, start_ns=0, end_ns=1) + if records is not None + else None + ) exporter_config = ExporterConfig( cfg=cfg, - results=None, + results=results, telemetry_results=None, - accuracy_results=summary, ) return AccuracyConsoleExporter(exporter_config=exporter_config) -def _task(passed: int, total: int, unparsed: int) -> TaskAccuracyStats: - return TaskAccuracyStats( - total=total, - passed=passed, - unparsed=unparsed, - accuracy_rate=passed / total if total else 0.0, - unparsed_rate=unparsed / total if total else 0.0, +def _make_metric(tag: str, correct: int, total: int, accuracy: float) -> MetricResult: + return MetricResult( + tag=tag, + header=tag, + unit="ratio", + sum=correct, + count=total, + current=accuracy, ) @pytest.mark.asyncio class TestAccuracyConsoleExporterExport: async def test_prints_table_with_task_and_overall_rows(self) -> None: - summary = AccuracySummary( - total_evaluated=10, - total_passed=8, - accuracy_rate=0.8, - overall_unparsed=1, - per_task={ - "algebra": _task(passed=3, total=5, unparsed=1), - "history": _task(passed=5, total=5, unparsed=0), - }, + exporter = _make_exporter( + records=[ + _make_metric("accuracy.overall", correct=8, total=10, accuracy=0.8), + _make_metric("accuracy.task.algebra", correct=3, total=5, accuracy=0.6), + _make_metric("accuracy.task.history", correct=5, total=5, accuracy=1.0), + _make_metric("accuracy.unparsed", correct=1, total=10, accuracy=0.1), + _make_metric( + "accuracy.unparsed.task.algebra", correct=1, total=5, accuracy=0.2 + ), + _make_metric( + "accuracy.unparsed.task.history", correct=0, total=5, accuracy=0.0 + ), + ] ) - exporter = _make_exporter(summary) buf = io.StringIO() console = Console(file=buf, highlight=False) await exporter.export(console) @@ -65,21 +73,35 @@ async def test_prints_table_with_task_and_overall_rows(self) -> None: assert "OVERALL" in output assert "Unparsed" in output - async def test_no_output_when_summary_is_none(self) -> None: - exporter = _make_exporter(None) + async def test_no_output_when_results_is_none(self) -> None: + exporter = _make_exporter(records=None) console = MagicMock() await exporter.export(console) console.print.assert_not_called() - async def test_overall_row_omitted_when_no_evaluations(self) -> None: - summary = AccuracySummary( - total_evaluated=0, - total_passed=0, - accuracy_rate=0.0, - overall_unparsed=0, - per_task={"algebra": _task(passed=3, total=5, unparsed=0)}, + async def test_no_output_when_records_is_none(self) -> None: + exporter = _make_exporter(records=None) + exporter.exporter_config.results = ProfileResults( + records=None, completed=0, start_ns=0, end_ns=1 + ) + console = MagicMock() + await exporter.export(console) + console.print.assert_not_called() + + async def test_no_output_when_no_accuracy_metrics(self) -> None: + exporter = _make_exporter( + records=[_make_metric("throughput", correct=0, total=100, accuracy=0.0)] + ) + console = MagicMock() + await exporter.export(console) + console.print.assert_not_called() + + async def test_overall_row_omitted_when_no_overall_metric(self) -> None: + exporter = _make_exporter( + records=[ + _make_metric("accuracy.task.algebra", correct=3, total=5, accuracy=0.6), + ] ) - exporter = _make_exporter(summary) buf = io.StringIO() console = Console(file=buf, highlight=False) await exporter.export(console) @@ -89,14 +111,11 @@ async def test_overall_row_omitted_when_no_evaluations(self) -> None: assert "algebra" in output async def test_accuracy_formatted_as_percentage(self) -> None: - summary = AccuracySummary( - total_evaluated=5, - total_passed=3, - accuracy_rate=0.6, - overall_unparsed=0, - per_task={"algebra": _task(passed=3, total=5, unparsed=0)}, + exporter = _make_exporter( + records=[ + _make_metric("accuracy.task.algebra", correct=3, total=5, accuracy=0.6), + ] ) - exporter = _make_exporter(summary) buf = io.StringIO() console = Console(file=buf, highlight=False) await exporter.export(console) @@ -104,18 +123,28 @@ async def test_accuracy_formatted_as_percentage(self) -> None: assert "60.00%" in buf.getvalue() async def test_warns_when_all_responses_unparsed(self) -> None: - """Smoke-test regression: when 100% of responses fail to parse, + """Smoke-test J regression: when 100% of responses fail to parse, the exporter must surface a loud diagnostic so users do not mistake mock-server / misconfigured-endpoint output for real accuracy=0% results.""" - summary = AccuracySummary( - total_evaluated=5, - total_passed=0, - accuracy_rate=0.0, - overall_unparsed=5, - per_task={"abstract_algebra": _task(passed=0, total=5, unparsed=5)}, + exporter = _make_exporter( + records=[ + _make_metric("accuracy.overall", correct=0, total=5, accuracy=0.0), + _make_metric( + "accuracy.task.abstract_algebra", + correct=0, + total=5, + accuracy=0.0, + ), + _make_metric("accuracy.unparsed", correct=5, total=5, accuracy=1.0), + _make_metric( + "accuracy.unparsed.task.abstract_algebra", + correct=5, + total=5, + accuracy=1.0, + ), + ] ) - exporter = _make_exporter(summary) buf = io.StringIO() console = Console(file=buf, highlight=False) await exporter.export(console) @@ -126,16 +155,18 @@ async def test_warns_when_all_responses_unparsed(self) -> None: assert "inference server" in output async def test_no_warning_when_partial_unparsed(self) -> None: - """Mixed parsed/unparsed runs are normal - the diagnostic must + """Mixed parsed/unparsed runs are normal — the diagnostic must only fire on the 100%-unparsed pathology.""" - summary = AccuracySummary( - total_evaluated=5, - total_passed=2, - accuracy_rate=0.4, - overall_unparsed=2, - per_task={"algebra": _task(passed=2, total=5, unparsed=2)}, + exporter = _make_exporter( + records=[ + _make_metric("accuracy.overall", correct=2, total=5, accuracy=0.4), + _make_metric("accuracy.task.algebra", correct=2, total=5, accuracy=0.4), + _make_metric("accuracy.unparsed", correct=2, total=5, accuracy=0.4), + _make_metric( + "accuracy.unparsed.task.algebra", correct=2, total=5, accuracy=0.4 + ), + ] ) - exporter = _make_exporter(summary) buf = io.StringIO() console = Console(file=buf, highlight=False) await exporter.export(console) @@ -143,19 +174,3 @@ async def test_no_warning_when_partial_unparsed(self) -> None: output = buf.getvalue() assert "Warning" not in output assert "inference server" not in output - - async def test_constructor_raises_when_accuracy_disabled(self) -> None: - cfg = make_benchmark_run( - model_names=["test-model"], - endpoint_type=EndpointType.COMPLETIONS, - streaming=False, - accuracy=None, - ).cfg - exporter_config = ExporterConfig( - cfg=cfg, - results=None, - telemetry_results=None, - accuracy_results=None, - ) - with pytest.raises(ConsoleExporterDisabled): - AccuracyConsoleExporter(exporter_config=exporter_config) diff --git a/tests/unit/accuracy/test_accuracy_data_exporter.py b/tests/unit/accuracy/test_accuracy_data_exporter.py index f4a0432180..6bf56e2f55 100644 --- a/tests/unit/accuracy/test_accuracy_data_exporter.py +++ b/tests/unit/accuracy/test_accuracy_data_exporter.py @@ -7,92 +7,108 @@ import pytest from aiperf.accuracy.accuracy_data_exporter import AccuracyDataExporter -from aiperf.accuracy.models import AccuracySummary, TaskAccuracyStats -from aiperf.common.exceptions import DataExporterDisabled +from aiperf.common.models import MetricResult +from aiperf.common.models.record_models import ProfileResults from aiperf.exporters.exporter_config import ExporterConfig from aiperf.plugin.enums import AccuracyBenchmarkType, EndpointType from tests.unit.conftest import make_benchmark_run -def _make_cfg(accuracy: dict | None = None): +def _make_cfg(): return make_benchmark_run( model_names=["test-model"], endpoint_type=EndpointType.CHAT, streaming=False, - accuracy=accuracy, + accuracy={"benchmark": AccuracyBenchmarkType.MMLU}, ).cfg -def _make_summary() -> AccuracySummary: - return AccuracySummary( - total_evaluated=10, - total_passed=8, - accuracy_rate=0.8, - overall_unparsed=1, - grader_name="mmlu", - per_task={ - "algebra": TaskAccuracyStats( - total=5, passed=3, unparsed=1, accuracy_rate=0.6, unparsed_rate=0.2 - ), - "history": TaskAccuracyStats( - total=5, passed=5, unparsed=0, accuracy_rate=1.0, unparsed_rate=0.0 - ), - }, - ) - - -def _make_exporter( - tmp_path: Path, summary: AccuracySummary | None -) -> AccuracyDataExporter: +def _make_exporter(tmp_path: Path, records: list[MetricResult]) -> AccuracyDataExporter: exporter_config = ExporterConfig( - cfg=_make_cfg({"benchmark": AccuracyBenchmarkType.MMLU}), - results=None, + cfg=_make_cfg(), + results=ProfileResults( + records=records, + completed=len(records), + start_ns=0, + end_ns=1, + ), telemetry_results=None, - accuracy_results=summary, ) exporter = AccuracyDataExporter(exporter_config=exporter_config) exporter._csv_path = tmp_path / "accuracy_results.csv" return exporter +def _make_metric(tag: str, correct: int, total: int, accuracy: float) -> MetricResult: + return MetricResult( + tag=tag, + header=tag, + unit="ratio", + sum=correct, + count=total, + current=accuracy, + ) + + @pytest.mark.asyncio class TestAccuracyDataExporterExport: - async def test_export_writes_task_rows_and_overall(self, tmp_path: Path) -> None: - exporter = _make_exporter(tmp_path, _make_summary()) + async def test_export_writes_overall_and_task_rows(self, tmp_path: Path) -> None: + records = [ + _make_metric("accuracy.overall", correct=8, total=10, accuracy=0.8), + _make_metric("accuracy.task.algebra", correct=3, total=5, accuracy=0.6), + _make_metric("accuracy.task.history", correct=5, total=5, accuracy=1.0), + _make_metric("accuracy.unparsed", correct=1, total=10, accuracy=0.1), + _make_metric( + "accuracy.unparsed.task.algebra", correct=1, total=5, accuracy=0.2 + ), + _make_metric( + "accuracy.unparsed.task.history", correct=0, total=5, accuracy=0.0 + ), + ] + exporter = _make_exporter(tmp_path, records) await exporter.export() rows = list(csv.reader(exporter._csv_path.open())) - assert rows[0] == [ - "task", - "total", - "passed", - "unparsed", - "accuracy_rate", - "unparsed_rate", + assert rows[0] == ["task", "correct", "total", "unparsed", "accuracy"] + assert rows[1] == ["OVERALL", "8", "10", "1", "0.8000"] + assert rows[2] == ["algebra", "3", "5", "1", "0.6000"] + assert rows[3] == ["history", "5", "5", "0", "1.0000"] + + async def test_export_skips_non_accuracy_metrics(self, tmp_path: Path) -> None: + records = [ + _make_metric("request_latency", correct=0, total=100, accuracy=0.0), + _make_metric("accuracy.overall", correct=4, total=10, accuracy=0.4), ] - # Per-task rows are sorted by name. - assert rows[1] == ["algebra", "5", "3", "1", "0.6", "0.2"] - assert rows[2] == ["history", "5", "5", "0", "1.0", "0.0"] - assert rows[3] == ["OVERALL", "10", "8", "1", "0.8", "0.1"] + exporter = _make_exporter(tmp_path, records) + + await exporter.export() + + rows = list(csv.reader(exporter._csv_path.open())) + assert len(rows) == 2 # header + overall only + assert rows[1][0] == "OVERALL" - async def test_export_does_nothing_when_summary_is_none( + async def test_export_does_nothing_when_no_accuracy_metrics( self, tmp_path: Path ) -> None: - exporter = _make_exporter(tmp_path, None) + records = [_make_metric("request_latency", correct=0, total=10, accuracy=0.0)] + exporter = _make_exporter(tmp_path, records) await exporter.export() assert not exporter._csv_path.exists() - async def test_constructor_raises_when_accuracy_disabled( + async def test_export_does_nothing_when_records_is_none( self, tmp_path: Path ) -> None: exporter_config = ExporterConfig( - cfg=_make_cfg(None), - results=None, + cfg=_make_cfg(), + results=ProfileResults(records=None, completed=0, start_ns=0, end_ns=1), telemetry_results=None, - accuracy_results=_make_summary(), ) - with pytest.raises(DataExporterDisabled): - AccuracyDataExporter(exporter_config=exporter_config) + exporter = AccuracyDataExporter(exporter_config=exporter_config) + exporter._csv_path = tmp_path / "accuracy_results.csv" + + await exporter.export() + + assert not exporter._csv_path.exists() diff --git a/tests/unit/accuracy/test_summary_to_metric_results.py b/tests/unit/accuracy/test_summary_to_metric_results.py new file mode 100644 index 0000000000..6fc71490fa --- /dev/null +++ b/tests/unit/accuracy/test_summary_to_metric_results.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from aiperf.accuracy.models import AccuracySummary, TaskAccuracyStats +from aiperf.common.enums import MetricConsoleGroup + + +def _make_summary() -> AccuracySummary: + return AccuracySummary( + total_evaluated=10, + total_passed=8, + accuracy_rate=0.8, + overall_unparsed=1, + grader_name="mmlu", + per_task={ + "history": TaskAccuracyStats( + total=5, passed=5, unparsed=0, accuracy_rate=1.0, unparsed_rate=0.0 + ), + "algebra": TaskAccuracyStats( + total=5, passed=3, unparsed=1, accuracy_rate=0.6, unparsed_rate=0.2 + ), + }, + ) + + +class TestToMetricResults: + def test_order_and_tags(self) -> None: + results = _make_summary().to_metric_results() + assert [r.tag for r in results] == [ + "accuracy.overall", + "accuracy.task.algebra", + "accuracy.task.history", + "accuracy.unparsed", + "accuracy.unparsed.task.algebra", + "accuracy.unparsed.task.history", + ] + + def test_overall_fields(self) -> None: + overall = _make_summary().to_metric_results()[0] + assert overall.header == "Accuracy (Overall)" + assert overall.unit == "ratio" + assert overall.count == 10 + assert overall.current == 0.8 + assert overall.sum == 8 + assert overall.console_group == MetricConsoleGroup.NONE + + def test_task_fields(self) -> None: + by_tag = {r.tag: r for r in _make_summary().to_metric_results()} + algebra = by_tag["accuracy.task.algebra"] + assert algebra.header == "Accuracy (algebra)" + assert algebra.unit == "ratio" + assert algebra.count == 5 + assert algebra.current == 0.6 + assert algebra.sum == 3 + assert algebra.console_group == MetricConsoleGroup.NONE + + def test_unparsed_overall_fields(self) -> None: + by_tag = {r.tag: r for r in _make_summary().to_metric_results()} + unparsed = by_tag["accuracy.unparsed"] + assert unparsed.header == "Accuracy Unparsed (Overall)" + assert unparsed.unit == "ratio" + assert unparsed.count == 10 + assert unparsed.current == 0.1 + assert unparsed.sum == 1 + assert unparsed.console_group == MetricConsoleGroup.NONE + + def test_unparsed_task_fields(self) -> None: + by_tag = {r.tag: r for r in _make_summary().to_metric_results()} + algebra = by_tag["accuracy.unparsed.task.algebra"] + assert algebra.header == "Accuracy Unparsed (algebra)" + assert algebra.count == 5 + assert algebra.current == 0.2 + assert algebra.sum == 1 + assert algebra.console_group == MetricConsoleGroup.NONE + + def test_json_shape_is_unit_count_sum(self) -> None: + """The perf JSON exporter projects each MetricResult through + JsonMetricResult (unit/avg/p*/count/sum only) and drops None fields, + yielding exactly ``{"unit":"ratio","count":N,"sum":M}`` for accuracy.*.""" + overall = _make_summary().to_metric_results()[0] + projected = overall.to_json_result().model_dump(exclude_none=True) + assert projected == {"unit": "ratio", "count": 10, "sum": 8} + + def test_empty_summary_emits_nothing(self) -> None: + empty = AccuracySummary( + total_evaluated=0, + total_passed=0, + accuracy_rate=0.0, + overall_unparsed=0, + ) + assert empty.to_metric_results() == [] diff --git a/tests/unit/controller/test_accuracy_shutdown_gate.py b/tests/unit/controller/test_accuracy_shutdown_gate.py index 7bc35b7df5..0aa8502adc 100644 --- a/tests/unit/controller/test_accuracy_shutdown_gate.py +++ b/tests/unit/controller/test_accuracy_shutdown_gate.py @@ -147,6 +147,53 @@ async def test_terminal_none_message_clears_flag_and_unblocks( controller.stop.assert_awaited_once() +class TestAccuracyResultsInjection: + """The dedicated-channel summary is materialized into the profile records + exactly once at export time so legacy exporters read ``accuracy.*``.""" + + def _controller_with_records(self, benchmark_run, mock_service_manager): + from aiperf.common.models.record_models import ( + ProcessRecordsResult, + ProfileResults, + ) + + controller = _build_controller( + benchmark_run, mock_service_manager, accuracy=True + ) + controller._profile_results = ProcessRecordsResult( + results=ProfileResults(records=[], completed=0, start_ns=0, end_ns=1), + ) + controller._accuracy_results = _summary() + return controller + + def test_injects_accuracy_metric_results_once( + self, benchmark_run, mock_service_manager + ) -> None: + controller = self._controller_with_records(benchmark_run, mock_service_manager) + + controller._inject_accuracy_results_into_records() + + records = controller._profile_results.results.records + tags = [r.tag for r in records] + assert tags == ["accuracy.overall", "accuracy.unparsed"] + assert controller._accuracy_results_injected is True + + # Re-export must not double-append. + controller._inject_accuracy_results_into_records() + assert [r.tag for r in controller._profile_results.results.records] == tags + + def test_no_injection_when_no_summary( + self, benchmark_run, mock_service_manager + ) -> None: + controller = self._controller_with_records(benchmark_run, mock_service_manager) + controller._accuracy_results = None + + controller._inject_accuracy_results_into_records() + + assert controller._profile_results.results.records == [] + assert controller._accuracy_results_injected is False + + class TestAccuracyShutdownGateDisabled: """Accuracy DISABLED: the accuracy term never blocks shutdown.""" From 877c12effece6af84611b703202db492243e66cd Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 21:22:13 -0700 Subject: [PATCH 21/39] refactor(accuracy): drop dead export wiring + fix architecture doc Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/architecture.md | 7 ++-- src/aiperf/accuracy/models.py | 38 -------------------- src/aiperf/controller/system_controller.py | 1 - src/aiperf/exporters/exporter_config.py | 2 -- src/aiperf/exporters/exporter_manager.py | 3 -- tests/unit/accuracy/test_accuracy_models.py | 40 --------------------- 6 files changed, 4 insertions(+), 87 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 651f534cc1..b186b7803e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -129,7 +129,7 @@ Accuracy benchmarking is one such dedicated channel, joining `metric_records`, ` The `AccuracyRecordProcessor` grades each parsed response against ground truth and returns an `AccuracyRecordsData` (`session_num`, `worker_id`, `benchmark_phase`, `timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, `confidence`, `expected`, `actual`, `reasoning`). The `RecordProcessorService` partitions these typed records off the metric-dict stream and pushes an `AccuracyRecordsMessage` to the Records Manager, which fans each record out to the `AccuracyAccumulator` (rolls records into an `AccuracySummary` of overall + per-task pass/unparsed rates) and the `AccuracyJSONLWriter` (streams the full per-response grading detail to `accuracy_export.jsonl`). -At end of the profiling phase, `RecordsManager._publish_accuracy_results(phase)` exports the phase-scoped `AccuracySummary` and publishes a `ProcessAccuracyResultMessage`. The `SystemController` receives it, stores the summary, and hands it to the exporters via `ExporterConfig.accuracy_results` — gated in shutdown like the telemetry and server-metrics results. The `AccuracyConsoleExporter` and `AccuracyDataExporter` render from that structured summary rather than filtering tagged metric results. +At end of the profiling phase, `RecordsManager._publish_accuracy_results(phase)` exports the phase-scoped `AccuracySummary` and publishes a `ProcessAccuracyResultMessage`. The `SystemController` receives it and stores the summary — gated in shutdown like the telemetry and server-metrics results. At export time, `SystemController._inject_accuracy_results_into_records` converts that summary back into legacy `accuracy.*` `MetricResult`s (via `AccuracySummary.to_metric_results()`) and appends them to `ProfileResults.records`. The `AccuracyConsoleExporter` and `AccuracyDataExporter` — like the main perf CSV/JSON exporters — then read those injected `accuracy.*` records. This inject bridge is what keeps the exported files (`accuracy_results.csv`, `profile_export_aiperf.{csv,json}`) byte-identical to the pre-refactor output. The dedicated per-response `accuracy_export.jsonl` is produced independently by the `AccuracyJSONLWriter` and is unaffected by this bridge. ```mermaid flowchart TD @@ -139,8 +139,9 @@ flowchart TD RM -->|process_record| JW[AccuracyJSONLWriter
accuracy_export.jsonl] ACC -->|export_results phase=PROFILING| PUB[_publish_accuracy_results] PUB -->|ProcessAccuracyResultMessage| SC[SystemController
stores AccuracySummary] - SC -->|ExporterConfig.accuracy_results| CE[AccuracyConsoleExporter] - SC -->|ExporterConfig.accuracy_results| DE[AccuracyDataExporter
accuracy_results.csv] + SC -->|_inject_accuracy_results_into_records
to_metric_results| REC[ProfileResults.records
accuracy.* MetricResults] + REC -->|read accuracy.* records| CE[AccuracyConsoleExporter] + REC -->|read accuracy.* records| DE[AccuracyDataExporter
accuracy_results.csv] ``` ### GPU Telemetry Manager diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index c305dab530..48c61d8c9f 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -171,44 +171,6 @@ class AccuracySummary(AIPerfBaseModel): default_factory=dict, description="Per-task accuracy rollups keyed by task name" ) - def to_json(self) -> dict[str, Any]: - """Return a plain-dict representation of the summary.""" - return self.model_dump() - - def to_csv(self) -> list[dict[str, Any]]: - """Return one row per task (sorted by name) plus a trailing OVERALL row. - - Columns: task, total, passed, unparsed, accuracy_rate, unparsed_rate. - Mirrors today's ``accuracy_results.csv`` shape (per-task rows + overall). - """ - rows: list[dict[str, Any]] = [ - { - "task": task, - "total": stats.total, - "passed": stats.passed, - "unparsed": stats.unparsed, - "accuracy_rate": stats.accuracy_rate, - "unparsed_rate": stats.unparsed_rate, - } - for task, stats in sorted(self.per_task.items()) - ] - overall_unparsed_rate = ( - self.overall_unparsed / self.total_evaluated - if self.total_evaluated - else 0.0 - ) - rows.append( - { - "task": "OVERALL", - "total": self.total_evaluated, - "passed": self.total_passed, - "unparsed": self.overall_unparsed, - "accuracy_rate": self.accuracy_rate, - "unparsed_rate": overall_unparsed_rate, - } - ) - return rows - def to_metric_results(self) -> list[MetricResult]: """Legacy ``accuracy.*`` MetricResult representation for byte-identical export. diff --git a/src/aiperf/controller/system_controller.py b/src/aiperf/controller/system_controller.py index c686e812ba..c1cf41d545 100644 --- a/src/aiperf/controller/system_controller.py +++ b/src/aiperf/controller/system_controller.py @@ -1145,7 +1145,6 @@ async def _print_post_benchmark_info_and_metrics(self) -> None: run=self.run, telemetry_results=self._telemetry_results, server_metrics_results=self._server_metrics_results, - accuracy_results=self._accuracy_results, ) # Export data files (CSV, JSON) with complete dataset including telemetry diff --git a/src/aiperf/exporters/exporter_config.py b/src/aiperf/exporters/exporter_config.py index 50d4cd712c..e3fd63f812 100644 --- a/src/aiperf/exporters/exporter_config.py +++ b/src/aiperf/exporters/exporter_config.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import TYPE_CHECKING -from aiperf.accuracy.models import AccuracySummary from aiperf.common.models import ProfileResults from aiperf.common.models.export_models import TelemetryExportData from aiperf.common.models.server_metrics_models import ServerMetricsResults @@ -23,7 +22,6 @@ class ExporterConfig: cfg: "BenchmarkConfig" telemetry_results: TelemetryExportData | None server_metrics_results: ServerMetricsResults | None = None - accuracy_results: AccuracySummary | None = None run: "BenchmarkRun | None" = None diff --git a/src/aiperf/exporters/exporter_manager.py b/src/aiperf/exporters/exporter_manager.py index 8965622a3c..4e3b690c58 100644 --- a/src/aiperf/exporters/exporter_manager.py +++ b/src/aiperf/exporters/exporter_manager.py @@ -7,7 +7,6 @@ from rich.console import Console -from aiperf.accuracy.models import AccuracySummary from aiperf.common.environment import Environment from aiperf.common.exceptions import ( ConsoleExporterDisabled, @@ -39,7 +38,6 @@ def __init__( run: "BenchmarkRun", telemetry_results: TelemetryExportData | None, server_metrics_results: ServerMetricsResults | None = None, - accuracy_results: AccuracySummary | None = None, **kwargs, ) -> None: super().__init__(**kwargs) @@ -51,7 +49,6 @@ def __init__( cfg=run.cfg, telemetry_results=telemetry_results, server_metrics_results=server_metrics_results, - accuracy_results=accuracy_results, run=run, ) diff --git a/tests/unit/accuracy/test_accuracy_models.py b/tests/unit/accuracy/test_accuracy_models.py index 4e31279400..250f28cc96 100644 --- a/tests/unit/accuracy/test_accuracy_models.py +++ b/tests/unit/accuracy/test_accuracy_models.py @@ -92,46 +92,6 @@ def _summary() -> AccuracySummary: ) -def test_summary_to_json_contains_keys() -> None: - data = _summary().to_json() - assert isinstance(data, dict) - assert data["accuracy_rate"] == 0.6 - assert "per_task" in data - - -def test_summary_to_csv_per_task_rows_sorted_with_overall_last() -> None: - rows = _summary().to_csv() - tasks = [row["task"] for row in rows] - assert tasks == ["mmlu.a", "mmlu.b", "OVERALL"] - overall = rows[-1] - assert overall["total"] == 5 - assert overall["passed"] == 3 - assert overall["accuracy_rate"] == 0.6 - assert overall["unparsed"] == 1 - first = rows[0] - assert set(first.keys()) == { - "task", - "total", - "passed", - "unparsed", - "accuracy_rate", - "unparsed_rate", - } - - -def test_summary_to_csv_empty_no_zero_division() -> None: - empty = AccuracySummary( - total_evaluated=0, - total_passed=0, - accuracy_rate=0.0, - overall_unparsed=0, - ) - rows = empty.to_csv() - assert rows[-1]["task"] == "OVERALL" - assert rows[-1]["accuracy_rate"] == 0.0 - assert rows[-1]["unparsed_rate"] == 0.0 - - def test_process_accuracy_result_defaults_none() -> None: assert ProcessAccuracyResult().results is None wrapped = ProcessAccuracyResult(results=_summary()) From 106f01be6b27fc3c2389642d1fa10791824b1b8f Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 21:27:52 -0700 Subject: [PATCH 22/39] feat(accuracy): capture full model output and thinking in per-record export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add model_output (the full answer content the model returned) and model_thinking (the model's reasoning_content channel, None when absent) to AccuracyRecordsData, populated by AccuracyRecordProcessor by splitting each response's content channel from its reasoning channel. These stream to accuracy_export.jsonl, so a graded record now carries what the model actually said and thought — not just the grader's extracted answer. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/accuracy/accuracy-benchmarking.md | 9 ++-- docs/accuracy/accuracy_stubs.md | 2 + .../accuracy/accuracy_record_processor.py | 33 ++++++++++++ src/aiperf/accuracy/models.py | 11 ++++ .../test_accuracy_record_processor.py | 51 +++++++++++++++++++ 5 files changed, 103 insertions(+), 3 deletions(-) diff --git a/docs/accuracy/accuracy-benchmarking.md b/docs/accuracy/accuracy-benchmarking.md index 821216ab21..e205969da5 100644 --- a/docs/accuracy/accuracy-benchmarking.md +++ b/docs/accuracy/accuracy-benchmarking.md @@ -363,9 +363,12 @@ unparsed_rate`. per graded response with the full grading detail that the summary rolls up. Each line carries: `session_num`, `worker_id`, `benchmark_phase`, `timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, `confidence`, -`expected` (ground truth), `actual` (extracted answer), and `reasoning`. Use it -for per-response post-hoc analysis — e.g. finding which prompts were graded -`unparsed` and why. +`expected` (ground truth), `actual` (extracted answer), `reasoning` (the +grader's decision trace), `model_output` (the full answer content the model +returned), and `model_thinking` (the model's reasoning/`reasoning_content` +channel when it emitted one, else `null`). Use it for per-response post-hoc +analysis — e.g. inspecting exactly what a reasoning model thought before an +`unparsed` answer. ## Architecture diff --git a/docs/accuracy/accuracy_stubs.md b/docs/accuracy/accuracy_stubs.md index 1c8ae952ca..2eaf7798b4 100644 --- a/docs/accuracy/accuracy_stubs.md +++ b/docs/accuracy/accuracy_stubs.md @@ -99,6 +99,8 @@ class AccuracyRecordsData(AIPerfBaseModel): expected: str # ground truth (from GradingResult.ground_truth) actual: str # extracted answer (from GradingResult.extracted_answer) reasoning: str # grader's explanation + model_output: str = "" # full answer content the model returned + model_thinking: str | None = None # model reasoning_content channel, if any ``` ### AccuracySummary diff --git a/src/aiperf/accuracy/accuracy_record_processor.py b/src/aiperf/accuracy/accuracy_record_processor.py index 1884fb7b19..2a8aeff0f0 100644 --- a/src/aiperf/accuracy/accuracy_record_processor.py +++ b/src/aiperf/accuracy/accuracy_record_processor.py @@ -112,6 +112,8 @@ async def process_record( self._log_grading_detail(metadata.session_num, response_text, result) + model_output, model_thinking = self._extract_output_and_thinking(record) + return AccuracyRecordsData( session_num=metadata.session_num, worker_id=metadata.worker_id, @@ -125,6 +127,8 @@ async def process_record( expected=result.ground_truth, actual=result.extracted_answer, reasoning=result.reasoning, + model_output=model_output, + model_thinking=model_thinking, ) def _log_grading_detail( @@ -169,3 +173,32 @@ def _extract_response_text(record: ParsedResponseRecord) -> str: if text: parts.append(text) return "".join(parts) + + @staticmethod + def _extract_output_and_thinking( + record: ParsedResponseRecord, + ) -> tuple[str, str | None]: + """Split the response into visible answer content and reasoning/thinking. + + ``model_output`` is the answer channel (``TextResponseData.text`` or + ``ReasoningResponseData.content``); ``model_thinking`` is the concatenated + ``reasoning_content`` from any ``ReasoningResponseData`` chunks, or None + when the model emitted no separate reasoning channel. + """ + output_parts: list[str] = [] + thinking_parts: list[str] = [] + for resp in record.content_responses: + data = resp.data + if data is None: + continue + reasoning = getattr(data, "reasoning", None) + if reasoning: + thinking_parts.append(reasoning) + content = getattr(data, "content", None) + if content is not None: + output_parts.append(content) + elif reasoning is None: + # Plain text (or tool-call) data with no reasoning channel. + output_parts.append(data.get_text()) + thinking = "".join(thinking_parts) if thinking_parts else None + return "".join(output_parts), thinking diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index 48c61d8c9f..edce7f355a 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -128,6 +128,17 @@ class AccuracyRecordsData(AIPerfBaseModel): "(maps from GradingResult.extracted_answer)" ) reasoning: str = Field(description="Grader's explanation of the grading decision") + model_output: str = Field( + default="", + description="Full model response content (the answer text the model " + "returned, excluding any separate reasoning channel). Always populated by " + "AccuracyRecordProcessor in production", + ) + model_thinking: str | None = Field( + default=None, + description="Model's reasoning/thinking content (reasoning_content) when " + "the model emitted a separate reasoning channel; None otherwise", + ) class TaskAccuracyStats(AIPerfBaseModel): diff --git a/tests/unit/accuracy/test_accuracy_record_processor.py b/tests/unit/accuracy/test_accuracy_record_processor.py index 0f1ea2f237..4646099545 100644 --- a/tests/unit/accuracy/test_accuracy_record_processor.py +++ b/tests/unit/accuracy/test_accuracy_record_processor.py @@ -159,6 +159,9 @@ async def test_process_record_maps_all_grading_and_metadata_fields( assert result.expected == "B" assert result.actual == "A" assert result.reasoning == "Wrong answer" + # Full model output captured; no separate reasoning channel here. + assert result.model_output == "Hello world" + assert result.model_thinking is None processor.grader.grade.assert_awaited_once_with("Hello world", "B") async def test_process_record_task_none_when_no_tasks( @@ -250,3 +253,51 @@ def test_non_verbose_debug_enabled_logs_at_debug(self, monkeypatch) -> None: processor._log_grading_detail(0, "some response", self._result()) assert len(logged) == 1 assert "sandboxed exec failed" in logged[0] + + +class TestExtractOutputAndThinking: + """`_extract_output_and_thinking` splits answer content from reasoning.""" + + @staticmethod + def _record(datas: list) -> "object": + from aiperf.common.models.record_models import ( + ParsedResponse, + ParsedResponseRecord, + ) + + record = MagicMock(spec=ParsedResponseRecord) + record.content_responses = [ + ParsedResponse(perf_ns=i, data=d) for i, d in enumerate(datas) + ] + return record + + def test_text_only_output_no_thinking(self) -> None: + from aiperf.common.models.record_models import TextResponseData + + record = self._record( + [TextResponseData(text="Hello"), TextResponseData(text=" world")] + ) + output, thinking = AccuracyRecordProcessor._extract_output_and_thinking(record) + assert output == "Hello world" + assert thinking is None + + def test_reasoning_split_into_output_and_thinking(self) -> None: + from aiperf.common.models.record_models import ReasoningResponseData + + record = self._record( + [ + ReasoningResponseData( + content="The answer is (B)", reasoning="Let me think... " + ), + ReasoningResponseData(content=" final.", reasoning="step two."), + ] + ) + output, thinking = AccuracyRecordProcessor._extract_output_and_thinking(record) + assert output == "The answer is (B) final." + assert thinking == "Let me think... step two." + + def test_empty_record_yields_empty_output_and_none_thinking(self) -> None: + record = self._record([]) + output, thinking = AccuracyRecordProcessor._extract_output_and_thinking(record) + assert output == "" + assert thinking is None From 651ddd78d3f22faade7667a49ee9c741c964ab45 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 21:30:29 -0700 Subject: [PATCH 23/39] feat(accuracy): add conversation_id + x_request_id to per-record export Add conversation_id (the stable problem id, which is the key into inputs.json for the full prompt) and x_request_id (unique per-request trace id) to AccuracyRecordsData, populated from the record metadata. A graded record in accuracy_export.jsonl can now be joined back to its exact prompt in inputs.json and to the raw records, without embedding the (potentially large, duplicated) prompt text inline. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/accuracy/accuracy-benchmarking.md | 6 ++++-- docs/accuracy/accuracy_stubs.md | 2 ++ src/aiperf/accuracy/accuracy_record_processor.py | 2 ++ src/aiperf/accuracy/models.py | 10 ++++++++++ tests/unit/accuracy/test_accuracy_record_processor.py | 4 ++++ 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/accuracy/accuracy-benchmarking.md b/docs/accuracy/accuracy-benchmarking.md index e205969da5..446ce21614 100644 --- a/docs/accuracy/accuracy-benchmarking.md +++ b/docs/accuracy/accuracy-benchmarking.md @@ -361,8 +361,10 @@ unparsed_rate`. **Per-record JSONL:** `/accuracy_export.jsonl` (or `_accuracy.jsonl` when an artifact prefix is configured) — one JSON line per graded response with the full grading detail that the summary rolls up. -Each line carries: `session_num`, `worker_id`, `benchmark_phase`, -`timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, `confidence`, +Each line carries: `session_num`, `conversation_id` (the problem id — the key to +look up the full prompt in `inputs.json`), `x_request_id`, `worker_id`, +`benchmark_phase`, `timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, +`confidence`, `expected` (ground truth), `actual` (extracted answer), `reasoning` (the grader's decision trace), `model_output` (the full answer content the model returned), and `model_thinking` (the model's reasoning/`reasoning_content` diff --git a/docs/accuracy/accuracy_stubs.md b/docs/accuracy/accuracy_stubs.md index 2eaf7798b4..5182072454 100644 --- a/docs/accuracy/accuracy_stubs.md +++ b/docs/accuracy/accuracy_stubs.md @@ -88,6 +88,8 @@ and `TelemetryRecord`. class AccuracyRecordsData(AIPerfBaseModel): record_type: ClassVar[str] = "accuracy" session_num: int # session/conversation index + conversation_id: str | None = None # problem id; key into inputs.json for the prompt + x_request_id: str | None = None # unique per-request id worker_id: str # record processor that produced this record benchmark_phase: CreditPhase # warmup vs profiling timestamp_ns: int # wall-clock ns when grading completed diff --git a/src/aiperf/accuracy/accuracy_record_processor.py b/src/aiperf/accuracy/accuracy_record_processor.py index 2a8aeff0f0..161e0f70e8 100644 --- a/src/aiperf/accuracy/accuracy_record_processor.py +++ b/src/aiperf/accuracy/accuracy_record_processor.py @@ -116,6 +116,8 @@ async def process_record( return AccuracyRecordsData( session_num=metadata.session_num, + conversation_id=metadata.conversation_id, + x_request_id=metadata.x_request_id, worker_id=metadata.worker_id, benchmark_phase=metadata.benchmark_phase, timestamp_ns=metadata.request_end_ns, diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index edce7f355a..5366df20ce 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -96,6 +96,16 @@ class AccuracyRecordsData(AIPerfBaseModel): ge=0, description="Conversation/session index this response came from, used to map to task", ) + conversation_id: str | None = Field( + default=None, + description="Stable id of the benchmark problem/conversation this response " + "answered; the key to look up the full prompt in inputs.json", + ) + x_request_id: str | None = Field( + default=None, + description="Unique per-request id (X-Request-ID) for tracing this exact " + "graded response back to the raw records", + ) worker_id: str = Field( description="ID of the record processor that produced this record" ) diff --git a/tests/unit/accuracy/test_accuracy_record_processor.py b/tests/unit/accuracy/test_accuracy_record_processor.py index 4646099545..6851139939 100644 --- a/tests/unit/accuracy/test_accuracy_record_processor.py +++ b/tests/unit/accuracy/test_accuracy_record_processor.py @@ -143,11 +143,15 @@ async def test_process_record_maps_all_grading_and_metadata_fields( worker_id="worker-9", request_end_ns=1_234_567_890, benchmark_phase=CreditPhase.PROFILING, + conversation_id="session_000004", + x_request_id="req-abc", ) result = await processor.process_record(sample_parsed_record, metadata) assert isinstance(result, AccuracyRecordsData) assert result.session_num == 4 + assert result.conversation_id == "session_000004" + assert result.x_request_id == "req-abc" assert result.worker_id == "worker-9" assert result.benchmark_phase == CreditPhase.PROFILING assert result.timestamp_ns == 1_234_567_890 From 90788c27f655dacc1be10d7526343532a4f6b287 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 21:35:56 -0700 Subject: [PATCH 24/39] refactor(accuracy): rename per-record reasoning field to explanation The AccuracyRecordsData field carrying the grader's decision trace was named `reasoning`, which collides with `model_thinking` (the model's own reasoning). Rename it to `explanation` for clarity. This field is new to this branch and has never appeared in any origin/main output file (the grader's reasoning was log-only upstream), so the rename breaks no shipped export schema. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/accuracy/accuracy-benchmarking.md | 9 +++++---- docs/accuracy/accuracy_stubs.md | 2 +- src/aiperf/accuracy/accuracy_record_processor.py | 2 +- src/aiperf/accuracy/models.py | 6 +++++- tests/unit/accuracy/test_accumulator.py | 2 +- tests/unit/accuracy/test_accuracy_models.py | 2 +- tests/unit/accuracy/test_accuracy_record_processor.py | 2 +- tests/unit/accuracy/test_jsonl_writer.py | 4 ++-- tests/unit/records/test_record_processor_service.py | 2 +- tests/unit/records/test_records_manager_accuracy.py | 2 +- 10 files changed, 19 insertions(+), 14 deletions(-) diff --git a/docs/accuracy/accuracy-benchmarking.md b/docs/accuracy/accuracy-benchmarking.md index 446ce21614..e064ce2e7f 100644 --- a/docs/accuracy/accuracy-benchmarking.md +++ b/docs/accuracy/accuracy-benchmarking.md @@ -365,10 +365,11 @@ Each line carries: `session_num`, `conversation_id` (the problem id — the key look up the full prompt in `inputs.json`), `x_request_id`, `worker_id`, `benchmark_phase`, `timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, `confidence`, -`expected` (ground truth), `actual` (extracted answer), `reasoning` (the -grader's decision trace), `model_output` (the full answer content the model -returned), and `model_thinking` (the model's reasoning/`reasoning_content` -channel when it emitted one, else `null`). Use it for per-response post-hoc +`expected` (ground truth), `actual` (extracted answer), `explanation` (the +**grader's** decision trace — why it scored the response the way it did), +`model_output` (the full answer content the model returned), and `model_thinking` +(the **model's** reasoning/`reasoning_content` channel when it emitted one, else +`null`). Use it for per-response post-hoc analysis — e.g. inspecting exactly what a reasoning model thought before an `unparsed` answer. diff --git a/docs/accuracy/accuracy_stubs.md b/docs/accuracy/accuracy_stubs.md index 5182072454..7cf5cabb53 100644 --- a/docs/accuracy/accuracy_stubs.md +++ b/docs/accuracy/accuracy_stubs.md @@ -100,7 +100,7 @@ class AccuracyRecordsData(AIPerfBaseModel): confidence: float # grading confidence 0.0-1.0 expected: str # ground truth (from GradingResult.ground_truth) actual: str # extracted answer (from GradingResult.extracted_answer) - reasoning: str # grader's explanation + explanation: str # grader's explanation of the score (NOT the model's) model_output: str = "" # full answer content the model returned model_thinking: str | None = None # model reasoning_content channel, if any ``` diff --git a/src/aiperf/accuracy/accuracy_record_processor.py b/src/aiperf/accuracy/accuracy_record_processor.py index 161e0f70e8..373c1872da 100644 --- a/src/aiperf/accuracy/accuracy_record_processor.py +++ b/src/aiperf/accuracy/accuracy_record_processor.py @@ -128,7 +128,7 @@ async def process_record( confidence=result.confidence, expected=result.ground_truth, actual=result.extracted_answer, - reasoning=result.reasoning, + explanation=result.reasoning, model_output=model_output, model_thinking=model_thinking, ) diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index 5366df20ce..15218fa090 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -137,7 +137,11 @@ class AccuracyRecordsData(AIPerfBaseModel): description="Answer extracted from the model response " "(maps from GradingResult.extracted_answer)" ) - reasoning: str = Field(description="Grader's explanation of the grading decision") + explanation: str = Field( + description="The grader's explanation of WHY it scored this response " + "correct/incorrect (distinct from model_thinking, which is the model's own " + "reasoning). Maps from GradingResult.reasoning", + ) model_output: str = Field( default="", description="Full model response content (the answer text the model " diff --git a/tests/unit/accuracy/test_accumulator.py b/tests/unit/accuracy/test_accumulator.py index dde032435e..3fcae8180a 100644 --- a/tests/unit/accuracy/test_accumulator.py +++ b/tests/unit/accuracy/test_accumulator.py @@ -47,7 +47,7 @@ def _record( confidence=1.0, expected="A", actual="A", - reasoning="matched", + explanation="matched", ) diff --git a/tests/unit/accuracy/test_accuracy_models.py b/tests/unit/accuracy/test_accuracy_models.py index 250f28cc96..50fced61c5 100644 --- a/tests/unit/accuracy/test_accuracy_models.py +++ b/tests/unit/accuracy/test_accuracy_models.py @@ -32,7 +32,7 @@ def _make_record(**overrides) -> AccuracyRecordsData: confidence=0.9, expected="B", actual="B", - reasoning="matched", + explanation="matched", ) defaults.update(overrides) return AccuracyRecordsData(**defaults) diff --git a/tests/unit/accuracy/test_accuracy_record_processor.py b/tests/unit/accuracy/test_accuracy_record_processor.py index 6851139939..db6da2bf65 100644 --- a/tests/unit/accuracy/test_accuracy_record_processor.py +++ b/tests/unit/accuracy/test_accuracy_record_processor.py @@ -162,7 +162,7 @@ async def test_process_record_maps_all_grading_and_metadata_fields( assert result.confidence == 0.42 assert result.expected == "B" assert result.actual == "A" - assert result.reasoning == "Wrong answer" + assert result.explanation == "Wrong answer" # Full model output captured; no separate reasoning channel here. assert result.model_output == "Hello world" assert result.model_thinking is None diff --git a/tests/unit/accuracy/test_jsonl_writer.py b/tests/unit/accuracy/test_jsonl_writer.py index 00801a8bd4..745a3c355a 100644 --- a/tests/unit/accuracy/test_jsonl_writer.py +++ b/tests/unit/accuracy/test_jsonl_writer.py @@ -27,7 +27,7 @@ def _record(*, timestamp_ns: int, task: str | None = "math") -> AccuracyRecordsD confidence=0.42, expected="A", actual="A", - reasoning="the answer is A", + explanation="the answer is A", ) @@ -65,6 +65,6 @@ async def test_records_round_trip(self) -> None: parsed = [orjson.loads(line) for line in lines] assert parsed[0]["timestamp_ns"] == 10 - assert parsed[0]["reasoning"] == "the answer is A" + assert parsed[0]["explanation"] == "the answer is A" assert parsed[0]["confidence"] == pytest.approx(0.42) assert parsed[1]["task"] == "algebra" diff --git a/tests/unit/records/test_record_processor_service.py b/tests/unit/records/test_record_processor_service.py index 73c3b8f8c7..6c11ad0872 100644 --- a/tests/unit/records/test_record_processor_service.py +++ b/tests/unit/records/test_record_processor_service.py @@ -32,7 +32,7 @@ def _make_accuracy_record(session_num: int = 0) -> AccuracyRecordsData: confidence=1.0, expected="A", actual="A", - reasoning="ok", + explanation="ok", ) diff --git a/tests/unit/records/test_records_manager_accuracy.py b/tests/unit/records/test_records_manager_accuracy.py index c2ba8ed932..28835c67b9 100644 --- a/tests/unit/records/test_records_manager_accuracy.py +++ b/tests/unit/records/test_records_manager_accuracy.py @@ -28,7 +28,7 @@ def _make_accuracy_record(session_num: int = 0) -> AccuracyRecordsData: confidence=1.0, expected="A", actual="A", - reasoning="ok", + explanation="ok", ) From 9a2a0163e352929af6ba5533993eb6bbfbd45b24 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 21:40:45 -0700 Subject: [PATCH 25/39] docs(accuracy): document per-record JSONL format and fix stale accuracy docs Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/accuracy/accuracy-benchmarking.md | 87 +++++++++++++++++++++----- docs/accuracy/accuracy_stubs.md | 12 +++- docs/architecture.md | 2 +- 3 files changed, 82 insertions(+), 19 deletions(-) diff --git a/docs/accuracy/accuracy-benchmarking.md b/docs/accuracy/accuracy-benchmarking.md index e064ce2e7f..da50b25450 100644 --- a/docs/accuracy/accuracy-benchmarking.md +++ b/docs/accuracy/accuracy-benchmarking.md @@ -308,8 +308,8 @@ aiperf profile my-model --url http://localhost:8000 \ | `multiple_choice` | A/B/C/D match against gold letter (lighteval `ExactMatches`). Under `--accuracy-enable-cot` the model emits a reasoning trace ending in `The answer is (X)`. | MMLU | | `mmlu_pro` | Extract the final `A`-`J` letter via the upstream 3-tier cascade: `answer is (X)` → `Answer: X` → last lone in-range letter. Fallback-tier or no-match responses are flagged `unparsed`. No optional dependencies. | MMLU-Pro | | `math` | Extract last `\boxed{...}`, fall back to "answer is X" / last number. Apply trt-llm `strip_string` normalization, then compare via `math_equal` (lowercase string → numeric `isclose` → symbolic equivalence via sympy + latex2sympy2-extended). | AIME | -| `exact_match` | Stub. | (unused) | -| `code_execution` | Stub. | (unused) | +| `exact_match` | Strict `pred.strip() == gold.strip()` — case-sensitive, no normalization (mirrors DeepEval `Scorer.exact_match_score`). Empty/whitespace-only response scores 0 and is flagged `unparsed`. | HellaSwag, BigBench-Hard | +| `code_execution` | pass@1 by executing the model's generated code against the benchmark's bundled public + private test cases via lighteval's `codegen_metrics` (sandboxed `ProcessPoolExecutor`, 6s per-test timeout). Extracts the code block with lighteval's `extract_code`; `correct` when pass@1 == 1.0, `unparsed` when no code block was extractable. Requires the `[accuracy]` extra (lighteval). | LiveCodeBench (`lcb_codegeneration`) | The `math` grader pipeline (aligned with `trt-llm-benchmark-recipe/src/accuracy/aime/`): @@ -358,20 +358,75 @@ expected format): trailing `OVERALL` row. Columns: `task, total, passed, unparsed, accuracy_rate, unparsed_rate`. -**Per-record JSONL:** `/accuracy_export.jsonl` (or -`_accuracy.jsonl` when an artifact prefix is configured) — one JSON line -per graded response with the full grading detail that the summary rolls up. -Each line carries: `session_num`, `conversation_id` (the problem id — the key to -look up the full prompt in `inputs.json`), `x_request_id`, `worker_id`, -`benchmark_phase`, `timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, -`confidence`, -`expected` (ground truth), `actual` (extracted answer), `explanation` (the -**grader's** decision trace — why it scored the response the way it did), -`model_output` (the full answer content the model returned), and `model_thinking` -(the **model's** reasoning/`reasoning_content` channel when it emitted one, else -`null`). Use it for per-response post-hoc -analysis — e.g. inspecting exactly what a reasoning model thought before an -`unparsed` answer. +### Per-record accuracy JSONL + +**Path:** `/accuracy_export.jsonl` by default, or +`_accuracy.jsonl` when an artifact prefix is configured (see +`AIPerfConfig.artifacts.accuracy_export_jsonl_file`). One JSON object per line, +one line per graded response — the full grading detail that the summary CSV and +console table roll up. Produced independently by the `AccuracyJSONLWriter`; it +is not affected by the summary/metric bridge that feeds the CSV and console. + +Each line is a serialized `AccuracyRecordsData` +(`src/aiperf/accuracy/models.py`) with these fields, in order: + +| Field | Type | Meaning | +|---|---|---| +| `session_num` | int | Conversation/session index this response came from | +| `conversation_id` | str \| null | Stable id of the benchmark problem/conversation; the key to look up the full prompt in `inputs.json` | +| `x_request_id` | str \| null | Unique per-request `X-Request-ID` for tracing this exact graded response back to the raw records | +| `worker_id` | str | Record processor that produced this record | +| `benchmark_phase` | str | Benchmark phase active when grading completed (`warmup` or `profiling`) | +| `timestamp_ns` | int | Nanosecond wall-clock timestamp when grading completed | +| `task` | str \| null | Accuracy task/subtask name (e.g. an MMLU subject); `null` when the dataset has no task label | +| `grader_name` | str | Which grader scored this response (e.g. `multiple_choice`) | +| `passed` | bool | Whether the response was graded correct | +| `unparsed` | bool | Whether the model output needed a regex fallback | +| `confidence` | float | Grading confidence (0.0–1.0) | +| `expected` | str | Ground-truth answer | +| `actual` | str | Answer extracted from the model response | +| `explanation` | str | The **grader's** explanation of why it scored the response correct/incorrect | +| `model_output` | str | The full answer content the model returned (the answer channel) | +| `model_thinking` | str \| null | The **model's** own reasoning (`reasoning_content`) when it emitted a separate reasoning channel; `null` otherwise | + +Three of these fields carry distinct text and are easy to conflate: + +- `explanation` — the **grader's** reasoning about the *score* (why it marked the + response right or wrong). +- `model_output` — the model's *answer* content (the answer channel). +- `model_thinking` — the model's own chain-of-thought / `reasoning_content` + channel, `null` when the model emitted no separate reasoning channel. + +The full prompt is **not** embedded in each record: it lives in `inputs.json` +keyed by `session_id`, which equals this record's `conversation_id`. Join on +that id to recover the prompt — this avoids duplicating multi-KB prompts on +every graded response. + +Example line (pretty-printed here; the file emits one compact object per line): + +```json +{ + "session_num": 0, + "conversation_id": "session_000000", + "x_request_id": "de56948f-8736-43e5-b636-303ebee20b20", + "worker_id": "worker_1c12efdd", + "benchmark_phase": "profiling", + "timestamp_ns": 1784176216352916652, + "task": "abstract_algebra", + "grader_name": "multiple_choice", + "passed": false, + "unparsed": false, + "confidence": 0.0, + "expected": "B", + "actual": "D", + "explanation": "first-line-of-response extracted to 'D'; ground_truth stripped to 'B'; match=False", + "model_output": "The answer is (D)", + "model_thinking": "I'll reason about each option in turn. Eliminating the implausible cases narrows it down. Therefore, The answer is (D)" +} +``` + +Use it for per-response post-hoc analysis — e.g. inspecting exactly what a +reasoning model thought before an `unparsed` answer. ## Architecture diff --git a/docs/accuracy/accuracy_stubs.md b/docs/accuracy/accuracy_stubs.md index 7cf5cabb53..960e212bdd 100644 --- a/docs/accuracy/accuracy_stubs.md +++ b/docs/accuracy/accuracy_stubs.md @@ -108,8 +108,16 @@ class AccuracyRecordsData(AIPerfBaseModel): ### AccuracySummary Structured accumulator result (replaces the old `list[MetricResult]`). -`AccuracyAccumulator` builds it; the console/CSV exporters render from it. -`to_csv()` emits per-task rows plus a trailing `OVERALL` row. +`AccuracyAccumulator` builds it and `RecordsManager` publishes it in a +`ProcessAccuracyResultMessage`. The accuracy exporters do **not** render from +the `AccuracySummary` directly: `SystemController` materializes it back into +legacy `accuracy.*` `MetricResult`s via `AccuracySummary.to_metric_results()` +and injects them into `ProfileResults.records` +(`SystemController._inject_accuracy_results_into_records`). The accuracy +CSV/console exporters and the main perf CSV/JSON then read those injected +`accuracy.*` records — this inject bridge is what keeps the exported files +byte-identical to the pre-refactor output. (`to_csv()`/`to_json()` were removed; +`to_metric_results()` is the only rendering path.) ```python class AccuracySummary(AIPerfBaseModel): diff --git a/docs/architecture.md b/docs/architecture.md index b186b7803e..801e657d6c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -127,7 +127,7 @@ Accuracy benchmarking is one such dedicated channel, joining `metric_records`, ` | `server_metrics` | Server Metrics Manager | `ServerMetricsRecordMessage` | `ServerMetricsAccumulator`, server-metrics JSONL writer | | `accuracy` | Record Processor (`AccuracyRecordProcessor`) | `AccuracyRecordsMessage` | `AccuracyAccumulator`, `AccuracyJSONLWriter` | -The `AccuracyRecordProcessor` grades each parsed response against ground truth and returns an `AccuracyRecordsData` (`session_num`, `worker_id`, `benchmark_phase`, `timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, `confidence`, `expected`, `actual`, `reasoning`). The `RecordProcessorService` partitions these typed records off the metric-dict stream and pushes an `AccuracyRecordsMessage` to the Records Manager, which fans each record out to the `AccuracyAccumulator` (rolls records into an `AccuracySummary` of overall + per-task pass/unparsed rates) and the `AccuracyJSONLWriter` (streams the full per-response grading detail to `accuracy_export.jsonl`). +The `AccuracyRecordProcessor` grades each parsed response against ground truth and returns an `AccuracyRecordsData` (`session_num`, `conversation_id`, `x_request_id`, `worker_id`, `benchmark_phase`, `timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, `confidence`, `expected`, `actual`, `explanation`, `model_output`, `model_thinking`). The `RecordProcessorService` partitions these typed records off the metric-dict stream and pushes an `AccuracyRecordsMessage` to the Records Manager, which fans each record out to the `AccuracyAccumulator` (rolls records into an `AccuracySummary` of overall + per-task pass/unparsed rates) and the `AccuracyJSONLWriter` (streams the full per-response grading detail to `accuracy_export.jsonl`, one JSON object per graded response). At end of the profiling phase, `RecordsManager._publish_accuracy_results(phase)` exports the phase-scoped `AccuracySummary` and publishes a `ProcessAccuracyResultMessage`. The `SystemController` receives it and stores the summary — gated in shutdown like the telemetry and server-metrics results. At export time, `SystemController._inject_accuracy_results_into_records` converts that summary back into legacy `accuracy.*` `MetricResult`s (via `AccuracySummary.to_metric_results()`) and appends them to `ProfileResults.records`. The `AccuracyConsoleExporter` and `AccuracyDataExporter` — like the main perf CSV/JSON exporters — then read those injected `accuracy.*` records. This inject bridge is what keeps the exported files (`accuracy_results.csv`, `profile_export_aiperf.{csv,json}`) byte-identical to the pre-refactor output. The dedicated per-response `accuracy_export.jsonl` is produced independently by the `AccuracyJSONLWriter` and is unaffected by this bridge. From dbf51483ff9ff4745d9aac184e9e2e943caca516 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 21:46:43 -0700 Subject: [PATCH 26/39] docs(accuracy): add the four lighteval_* graders to the grader table The main grader table in accuracy-benchmarking.md listed only 5 of the 9 graders; add lighteval_expr (AIME24/25), lighteval_latex (MATH-500), lighteval_gpqa (GPQA-Diamond), and lighteval_gsm8k (GSM8K) so it matches the benchmarks table and the registered plugins. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/accuracy/accuracy-benchmarking.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/accuracy/accuracy-benchmarking.md b/docs/accuracy/accuracy-benchmarking.md index da50b25450..e6386a9c49 100644 --- a/docs/accuracy/accuracy-benchmarking.md +++ b/docs/accuracy/accuracy-benchmarking.md @@ -310,6 +310,10 @@ aiperf profile my-model --url http://localhost:8000 \ | `math` | Extract last `\boxed{...}`, fall back to "answer is X" / last number. Apply trt-llm `strip_string` normalization, then compare via `math_equal` (lowercase string → numeric `isclose` → symbolic equivalence via sympy + latex2sympy2-extended). | AIME | | `exact_match` | Strict `pred.strip() == gold.strip()` — case-sensitive, no normalization (mirrors DeepEval `Scorer.exact_match_score`). Empty/whitespace-only response scores 0 and is flagged `unparsed`. | HellaSwag, BigBench-Hard | | `code_execution` | pass@1 by executing the model's generated code against the benchmark's bundled public + private test cases via lighteval's `codegen_metrics` (sandboxed `ProcessPoolExecutor`, 6s per-test timeout). Extracts the code block with lighteval's `extract_code`; `correct` when pass@1 == 1.0, `unparsed` when no code block was extractable. Requires the `[accuracy]` extra (lighteval). | LiveCodeBench (`lcb_codegeneration`) | +| `lighteval_expr` | Sympy-backed expression extraction and symbolic equivalence (lighteval `expr_gold_metric`): pulls the model's final expression and compares it to gold via lighteval's math parser. Requires the `[accuracy]` extra (lighteval). | AIME24, AIME25 | +| `lighteval_latex` | Same as `lighteval_expr` but the gold/prediction extractor uses lighteval's `LatexExtractionConfig` for `\boxed{...}` LaTeX answers (lighteval `latex_gold_metric`). Requires the `[accuracy]` extra. | MATH-500 | +| `lighteval_gpqa` | Multiple-choice `A`-`D` index extraction via lighteval's `gpqa_metric` (`NativeLetters`), using the simple-evals template the GPQA-Diamond loader mirrors for parity. Requires the `[accuracy]` extra. | GPQA-Diamond | +| `lighteval_gsm8k` | Extract the number after `####` from gold and the last number from the prediction (preferring a `####` marker when present); numeric comparison so `24` and `24.0` match (lighteval `quasi_exact_match_gsm8k`). Pure-regex — no lighteval install required. | GSM8K | The `math` grader pipeline (aligned with `trt-llm-benchmark-recipe/src/accuracy/aime/`): From e7e61d42dbc1eceaa1ef38594c69c1cbe5c9a5de Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 22:52:22 -0700 Subject: [PATCH 27/39] refactor(records): 2-stage producers/observers + generic RecordsMessage Replace the per-type record message classes and the hardcoded channel builder map with one generic per-request envelope. Producers (stage 1) emit finished typed records grouped by their declared record_type; observers (stage 2) view them via RecordObserverContext; the record processor ships exactly one RecordsMessage carrying every produced record plus the request metadata/error. The records manager dispatches each record generically (no isinstance) and drives the per-request lockstep off the envelope via records_tracker.update_from_request. The metric producer now returns a finished MetricRecordsData (metrics + trace_data + error), and the mid-run cache-reporting hint moves into MetricsAccumulator so the reader needs zero metric-record inspection. Delete MetricRecordsMessage (and to_data) and AccuracyRecordsMessage. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/plugins/plugin-system.md | 5 +- src/aiperf/common/enums/enums.py | 1 + src/aiperf/common/messages/__init__.py | 6 +- .../common/messages/accuracy_messages.py | 20 +- .../common/messages/inference_messages.py | 61 ++--- src/aiperf/common/models/record_models.py | 4 +- src/aiperf/metrics/accumulator.py | 21 ++ src/aiperf/plugin/categories.yaml | 15 +- src/aiperf/plugin/enums.py | 8 +- src/aiperf/plugin/plugins.py | 8 +- src/aiperf/plugin/plugins.yaml | 32 ++- src/aiperf/plugin/schema/plugins.schema.json | 56 +++- src/aiperf/plugin/schema/schemas.py | 17 ++ .../metric_record_processor.py | 18 +- .../outputs_json_record_processor.py | 16 +- src/aiperf/post_processors/protocols.py | 17 +- .../raw_record_writer_processor.py | 9 +- .../record_observer_context.py | 40 +++ .../records/record_processor_service.py | 196 ++++++++------ src/aiperf/records/records_manager.py | 70 ++--- src/aiperf/records/records_tracker.py | 23 +- tests/unit/accuracy/test_accuracy_models.py | 41 +-- tests/unit/post_processors/conftest.py | 27 +- .../test_metric_record_processor.py | 52 ++-- .../test_metric_results_strategy.py | 10 +- .../test_metrics_accumulator.py | 254 +++++++++--------- .../test_network_adjusted_metrics.py | 10 +- .../test_otel_metrics_results_processor.py | 18 +- .../test_outputs_json_record_processor.py | 13 +- .../test_post_processor_integration.py | 22 +- .../test_raw_record_writer_adversarial.py | 28 +- .../test_raw_record_writer_processor.py | 37 ++- ...est_record_export_jsonl_writer_detailed.py | 68 ++--- ...test_timeslice_metric_results_processor.py | 38 +-- .../test_timing_results_strategy.py | 8 +- .../property/_numeric_bounds_baseline.txt | 4 +- .../test_realtime_metrics_generation.py | 10 +- .../records/test_record_processor_service.py | 161 +++++++---- tests/unit/records/test_records_manager.py | 105 ++++---- .../records/test_records_manager_accuracy.py | 60 +++-- 40 files changed, 927 insertions(+), 682 deletions(-) create mode 100644 src/aiperf/post_processors/record_observer_context.py diff --git a/docs/plugins/plugin-system.md b/docs/plugins/plugin-system.md index fd68ae8ce5..838ec32eb2 100644 --- a/docs/plugins/plugin-system.md +++ b/docs/plugins/plugin-system.md @@ -100,7 +100,7 @@ for entry, cls in plugins.iter_all(PluginType.ENDPOINT): ## Plugin Categories -AIPerf supports 33 plugin categories organized by function, including `api_router` and `public_dataset_loader`: +AIPerf supports 34 plugin categories organized by function, including `api_router` and `public_dataset_loader`: ### Timing Categories @@ -133,7 +133,8 @@ AIPerf supports 33 plugin categories organized by function, including `api_route | Category | Enum | Description | |----------|------|-------------| -| `record_processor` | `RecordProcessorType` | Per-record metric computation | +| `record_processor` | `RecordProcessorType` | Record PRODUCERS: parse a record and emit one finished typed record on a declared record-type channel (stage 1) | +| `record_observer` | `RecordObserverType` | Record OBSERVERS: view the produced records + the record and act (e.g. write JSONL); emit no channel record (stage 2) | | `accumulator` | `AccumulatorType` | Record-type-routed aggregation and summary computation | | `analyzer` | `AnalyzerType` | Summarize-time cross-accumulator joins (e.g. energy efficiency), reading peers via `SummaryContext` | | `stream_exporter` | `StreamExporterType` | Record-type-routed streaming sinks such as JSONL and OpenTelemetry | diff --git a/src/aiperf/common/enums/enums.py b/src/aiperf/common/enums/enums.py index 32f6874aa8..c292ef6914 100644 --- a/src/aiperf/common/enums/enums.py +++ b/src/aiperf/common/enums/enums.py @@ -363,6 +363,7 @@ class MessageType(CaseInsensitiveStrEnum): HEARTBEAT = "heartbeat" INFERENCE_RESULTS = "inference_results" METRIC_RECORDS = "metric_records" + RECORDS = "records" PARSED_INFERENCE_RESULTS = "parsed_inference_results" PROCESSING_STATS = "processing_stats" PROCESS_RECORDS_RESULT = "process_records_result" diff --git a/src/aiperf/common/messages/__init__.py b/src/aiperf/common/messages/__init__.py index 7fe39db898..9f7375e3ce 100644 --- a/src/aiperf/common/messages/__init__.py +++ b/src/aiperf/common/messages/__init__.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 from aiperf.common.messages.accuracy_messages import ( - AccuracyRecordsMessage, ProcessAccuracyResultMessage, ) from aiperf.common.messages.base_messages import ( @@ -43,8 +42,8 @@ from aiperf.common.messages.inference_messages import ( InferenceResultsMessage, MetricRecordsData, - MetricRecordsMessage, RealtimeMetricsMessage, + RecordsMessage, ) from aiperf.common.messages.network_latency_messages import ( NetworkLatencyRecordMessage, @@ -81,7 +80,6 @@ ) __all__ = [ - "AccuracyRecordsMessage", "AllRecordsReceivedMessage", "BaseServiceErrorMessage", "BaseServiceMessage", @@ -104,7 +102,6 @@ "InferenceResultsMessage", "Message", "MetricRecordsData", - "MetricRecordsMessage", "NetworkLatencyRecordMessage", "ProcessRecordsCommand", "ProcessRecordsResponse", @@ -121,6 +118,7 @@ "RealtimeMetricsCommand", "RealtimeMetricsMessage", "RealtimeTelemetryMetricsMessage", + "RecordsMessage", "RecordsProcessingStatsMessage", "RegisterServiceCommand", "RegistrationMessage", diff --git a/src/aiperf/common/messages/accuracy_messages.py b/src/aiperf/common/messages/accuracy_messages.py index 80f441fa1b..64da9033ec 100644 --- a/src/aiperf/common/messages/accuracy_messages.py +++ b/src/aiperf/common/messages/accuracy_messages.py @@ -4,30 +4,12 @@ from pydantic import Field -from aiperf.accuracy.models import AccuracyRecordsData, ProcessAccuracyResult +from aiperf.accuracy.models import ProcessAccuracyResult from aiperf.common.enums import MessageType from aiperf.common.messages.service_messages import BaseServiceMessage -from aiperf.common.models import ErrorDetails from aiperf.common.types import MessageTypeT -class AccuracyRecordsMessage(BaseServiceMessage): - """Message from a record processor to the records manager carrying graded - accuracy records on the dedicated ``accuracy`` channel. - """ - - message_type: MessageTypeT = MessageType.ACCURACY_RECORD - - records: list[AccuracyRecordsData] = Field( - default_factory=list, - description="The graded accuracy records produced by the record processor", - ) - error: ErrorDetails | None = Field( - default=None, - description="The error details if grading/record production failed.", - ) - - class ProcessAccuracyResultMessage(BaseServiceMessage): """Message containing a processed accuracy summary - mirrors ProcessServerMetricsResultMessage.""" diff --git a/src/aiperf/common/messages/inference_messages.py b/src/aiperf/common/messages/inference_messages.py index ea0eb4d60a..67657d40ae 100644 --- a/src/aiperf/common/messages/inference_messages.py +++ b/src/aiperf/common/messages/inference_messages.py @@ -5,7 +5,6 @@ from pydantic import Field, SerializeAsAny, field_validator -from aiperf.common.aiperf_logger import AIPerfLogger from aiperf.common.enums import MessageType, MetricValueTypeT from aiperf.common.messages.service_messages import BaseServiceMessage from aiperf.common.models import ErrorDetails, RequestRecord @@ -14,8 +13,6 @@ from aiperf.common.models.trace_models import BaseTraceData from aiperf.common.types import MessageTypeT, MetricTagT -_logger = AIPerfLogger(__name__) - class InferenceResultsMessage(BaseServiceMessage): """Message for a inference results.""" @@ -62,60 +59,36 @@ def valid(self) -> bool: return self.error is None -class MetricRecordsMessage(BaseServiceMessage): - """Message from the result parser to the records manager to notify it - of the metric records for a single request.""" +class RecordsMessage(BaseServiceMessage): + """Per-request envelope from the record processor service to the records manager. + + One ``RecordsMessage`` is pushed for every inference record (the credit + lockstep contract). It carries the request metadata plus the flattened list + of typed records produced for that request. Each record self-identifies via + its own ``record_type`` ClassVar, so the records manager dispatches + generically without inspecting the message for a record type. + """ - message_type: MessageTypeT = MessageType.METRIC_RECORDS + message_type: MessageTypeT = MessageType.RECORDS metadata: MetricRecordMetadata = Field( - ..., description="The metadata of the request record." + ..., description="The metadata of the request record; drives the lockstep." ) - results: list[dict[MetricTagT, MetricValueTypeT]] = Field( - ..., description="The record processor metric results" - ) - trace_data: SerializeAsAny[BaseTraceData] | None = Field( - default=None, - description="Comprehensive trace data captured via a trace config. " - "Includes detailed timing for connection establishment, DNS resolution, request/response events, etc. " - "The type of the trace data is determined by the transport and library used.", + records: list[SerializeAsAny[Any]] = Field( + default_factory=list, + description="The typed records produced for this request. Each record " + "self-identifies via its own record_type ClassVar.", ) error: ErrorDetails | None = Field( - default=None, description="The error details if the request failed." + default=None, + description="The request-level error details if the request failed.", ) - @field_validator("trace_data", mode="before") - @classmethod - def route_trace_data(cls, v: Any) -> BaseTraceData | None: - """Route nested trace_data to correct subclass based on trace_type discriminator.""" - if isinstance(v, dict): - return BaseTraceData.from_json(v) - return v - @property def valid(self) -> bool: """Whether the request was valid.""" return self.error is None - def to_data(self) -> MetricRecordsData: - """Convert the metric records message to a MetricRecordsData for processing by the records manager.""" - metrics = {} - for result in self.results: - for tag, value in result.items(): - if tag in metrics: - _logger.warning( - f"Duplicate metric tag '{tag}' found in results. " - f"Overwriting previous value {metrics[tag]} with {value}." - ) - metrics[tag] = value - - return MetricRecordsData( - metadata=self.metadata, - metrics=metrics, - trace_data=self.trace_data, - error=self.error, - ) - class RealtimeMetricsMessage(BaseServiceMessage): """Message from the records manager to show real-time metrics for the profile run.""" diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index fb34dceaa7..52eafd989c 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -1296,7 +1296,7 @@ class MetricRecordInfo(AIPerfBaseModel): metadata: MetricRecordMetadata = Field( ..., - description="The metadata of the record. Should match the metadata in the MetricRecordsMessage.", + description="The metadata of the record. Should match the metadata in the RecordsMessage.", ) metrics: dict[str, MetricValue] = Field( ..., @@ -1319,7 +1319,7 @@ class RawRecordInfo(AIPerfBaseModel): metadata: MetricRecordMetadata = Field( ..., - description="The metadata of the record. Should match the metadata in the MetricRecordsMessage.", + description="The metadata of the record. Should match the metadata in the RecordsMessage.", ) start_perf_ns: int = Field( default_factory=time.perf_counter_ns, diff --git a/src/aiperf/metrics/accumulator.py b/src/aiperf/metrics/accumulator.py index 1b4fea1493..0fadf07b40 100644 --- a/src/aiperf/metrics/accumulator.py +++ b/src/aiperf/metrics/accumulator.py @@ -26,6 +26,10 @@ from aiperf.metrics.accumulator_models import AccumulatorMetricsSummary from aiperf.metrics.accumulator_sweeps import compute_sweep_curves from aiperf.metrics.base_metric import BaseMetric +from aiperf.metrics.cache_reporting_hint import ( + CACHE_REPORTING_HINT, + usage_without_cache_in_record, +) from aiperf.metrics.column_store import ColumnStore from aiperf.metrics.derived_latency import ( inject_adjusted_latency_metrics, @@ -104,6 +108,9 @@ def __init__( # emitted by inject_replay_sched_lag_metrics (fires at most once per run). self._replay_degraded_warned: bool = False + # One-shot latch for the mid-run "enable cache reporting" server-knob hint. + self._warned_missing_cache_reporting: bool = False + # Derive functions for DERIVED metrics # _setup_metrics includes transitive dependencies (RECORD/AGGREGATE), # so filter to only metrics that actually have derive_value. @@ -161,6 +168,7 @@ def record_count(self) -> int: async def process_record(self, record: MetricRecordsData) -> None: """Ingest a single ``MetricRecordsData`` into columnar storage.""" + self._maybe_hint_missing_cache_reporting(record) idx = self._next_record_idx self._next_record_idx += 1 meta = record.metadata @@ -206,6 +214,19 @@ async def process_record(self, record: MetricRecordsData) -> None: }, ) + def _maybe_hint_missing_cache_reporting(self, record: MetricRecordsData) -> None: + """Warn once, mid-run, when the server reports token usage but no prompt-cache + reads — the signature of a cache-capable server that hasn't been told to + report ``cached_tokens``. Fires on the first qualifying record so a long run + can be aborted and re-launched with the flag set; the end-of-run console + exporter emits the same hint for anyone who only reads the final summary. + """ + if self._warned_missing_cache_reporting: + return + if usage_without_cache_in_record(record.metrics): + self._warned_missing_cache_reporting = True + self.warning(CACHE_REPORTING_HINT) + def query_time_range(self, start_ns: int, end_ns: int) -> BoolArray: """Return a boolean mask where True marks records in [start_ns, end_ns).""" n = self._column_store.count diff --git a/src/aiperf/plugin/categories.yaml b/src/aiperf/plugin/categories.yaml index 536eda4036..6310d4eb56 100644 --- a/src/aiperf/plugin/categories.yaml +++ b/src/aiperf/plugin/categories.yaml @@ -146,11 +146,20 @@ transport: record_processor: protocol: aiperf.post_processors.protocols:RecordProcessorProtocol + metadata_class: aiperf.plugin.schema.schemas:RecordProducerMetadata enum: RecordProcessorType description: | - Record processors stream records and compute metrics in a distributed manner. - First stage of metrics pipeline, handling per-record computations. - One-to-many mapping: multiple processors can be loaded simultaneously. + Record PRODUCERS parse a record and emit one typed result on a declared + record-type channel. First stage of the metrics pipeline, handling per-record + computations. One-to-many mapping: multiple producers can be loaded simultaneously. + +record_observer: + protocol: aiperf.post_processors.protocols:RecordObserverProtocol + enum: RecordObserverType + description: | + Record OBSERVERS view the produced results + the record and act (e.g. write + JSONL); they return nothing and emit no channel record. Second stage of the + record-processor service. One-to-many mapping: multiple observers loaded simultaneously. accumulator: protocol: aiperf.common.accumulator_protocols:AccumulatorProtocol diff --git a/src/aiperf/plugin/enums.py b/src/aiperf/plugin/enums.py index 6c9952a8ca..4685250d95 100644 --- a/src/aiperf/plugin/enums.py +++ b/src/aiperf/plugin/enums.py @@ -12,7 +12,7 @@ from aiperf.plugin import plugins from aiperf.plugin.extensible_enums import create_enum -__all__ = ["APIRouterType", "APIRouterTypeStr", "AccumulatorType", "AccumulatorTypeStr", "AccuracyBenchmarkType", "AccuracyBenchmarkTypeStr", "AccuracyGraderType", "AccuracyGraderTypeStr", "AnalyzerType", "AnalyzerTypeStr", "ArrivalPattern", "ArrivalPatternStr", "CommClientType", "CommClientTypeStr", "CommunicationBackend", "CommunicationBackendStr", "ComposerType", "ComposerTypeStr", "ConsoleExporterType", "ConsoleExporterTypeStr", "ConvergenceCriterionType", "ConvergenceCriterionTypeStr", "CustomDatasetType", "CustomDatasetTypeStr", "DataExporterType", "DataExporterTypeStr", "DatasetBackingStoreType", "DatasetBackingStoreTypeStr", "DatasetClientStoreType", "DatasetClientStoreTypeStr", "DatasetFormat", "DatasetFormatStr", "DatasetSamplingStrategy", "DatasetSamplingStrategyStr", "EndpointType", "EndpointTypeStr", "GPUTelemetryCollectorType", "GPUTelemetryCollectorTypeStr", "PhaseType", "PhaseTypeStr", "PlotType", "PlotTypeStr", "PluginType", "PluginTypeStr", "PublicDatasetType", "PublicDatasetTypeStr", "RampType", "RampTypeStr", "RecordProcessorType", "RecordProcessorTypeStr", "SearchPlannerType", "SearchPlannerTypeStr", "SearchRecipePostProcessType", "SearchRecipePostProcessTypeStr", "SearchRecipeType", "SearchRecipeTypeStr", "ServiceRunType", "ServiceRunTypeStr", "ServiceType", "ServiceTypeStr", "StreamExporterType", "StreamExporterTypeStr", "TimingMode", "TimingModeStr", "TransportType", "TransportTypeStr", "UIType", "UITypeStr", "URLSelectionStrategy", "URLSelectionStrategyStr", "ZMQProxyType", "ZMQProxyTypeStr"] +__all__ = ["APIRouterType", "APIRouterTypeStr", "AccumulatorType", "AccumulatorTypeStr", "AccuracyBenchmarkType", "AccuracyBenchmarkTypeStr", "AccuracyGraderType", "AccuracyGraderTypeStr", "AnalyzerType", "AnalyzerTypeStr", "ArrivalPattern", "ArrivalPatternStr", "CommClientType", "CommClientTypeStr", "CommunicationBackend", "CommunicationBackendStr", "ComposerType", "ComposerTypeStr", "ConsoleExporterType", "ConsoleExporterTypeStr", "ConvergenceCriterionType", "ConvergenceCriterionTypeStr", "CustomDatasetType", "CustomDatasetTypeStr", "DataExporterType", "DataExporterTypeStr", "DatasetBackingStoreType", "DatasetBackingStoreTypeStr", "DatasetClientStoreType", "DatasetClientStoreTypeStr", "DatasetFormat", "DatasetFormatStr", "DatasetSamplingStrategy", "DatasetSamplingStrategyStr", "EndpointType", "EndpointTypeStr", "GPUTelemetryCollectorType", "GPUTelemetryCollectorTypeStr", "PhaseType", "PhaseTypeStr", "PlotType", "PlotTypeStr", "PluginType", "PluginTypeStr", "PublicDatasetType", "PublicDatasetTypeStr", "RampType", "RampTypeStr", "RecordObserverType", "RecordObserverTypeStr", "RecordProcessorType", "RecordProcessorTypeStr", "SearchPlannerType", "SearchPlannerTypeStr", "SearchRecipePostProcessType", "SearchRecipePostProcessTypeStr", "SearchRecipeType", "SearchRecipeTypeStr", "ServiceRunType", "ServiceRunTypeStr", "ServiceType", "ServiceTypeStr", "StreamExporterType", "StreamExporterTypeStr", "TimingMode", "TimingModeStr", "TransportType", "TransportTypeStr", "UIType", "UITypeStr", "URLSelectionStrategy", "URLSelectionStrategyStr", "ZMQProxyType", "ZMQProxyTypeStr"] # Plugin Protocol Categories if TYPE_CHECKING: @@ -75,7 +75,11 @@ RecordProcessorTypeStr: TypeAlias = str RecordProcessorType = plugins.create_enum(PluginType.RECORD_PROCESSOR, "RecordProcessorType", module=__name__) -"""Dynamic enum for record processor. Example: RecordProcessorType.ACCURACY_RECORD, RecordProcessorType.OUTPUTS_JSON, RecordProcessorType.RAW_RECORD_WRITER""" +"""Dynamic enum for record processor. Example: RecordProcessorType.ACCURACY_RECORD, RecordProcessorType.METRIC_RECORD""" + +RecordObserverTypeStr: TypeAlias = str +RecordObserverType = plugins.create_enum(PluginType.RECORD_OBSERVER, "RecordObserverType", module=__name__) +"""Dynamic enum for record observer. Example: RecordObserverType.OUTPUTS_JSON, RecordObserverType.RAW_RECORD_WRITER""" AccumulatorTypeStr: TypeAlias = str AccumulatorType = plugins.create_enum(PluginType.ACCUMULATOR, "AccumulatorType", module=__name__) diff --git a/src/aiperf/plugin/plugins.py b/src/aiperf/plugin/plugins.py index a624bc7f28..2d6e5b805f 100644 --- a/src/aiperf/plugin/plugins.py +++ b/src/aiperf/plugin/plugins.py @@ -939,8 +939,8 @@ def _load_package_metadata( from aiperf.orchestrator.convergence.base import ConvergenceCriterion from aiperf.orchestrator.search_planner.base import SearchPlanner from aiperf.plot.core.plot_type_handlers import PlotTypeHandlerProtocol - from aiperf.plugin.enums import APIRouterType, AccumulatorType, AccuracyBenchmarkType, AccuracyGraderType, AnalyzerType, ArrivalPattern, CommClientType, CommunicationBackend, ComposerType, ConsoleExporterType, ConvergenceCriterionType, CustomDatasetType, DataExporterType, DatasetBackingStoreType, DatasetClientStoreType, DatasetSamplingStrategy, EndpointType, GPUTelemetryCollectorType, PlotType, PluginType, PluginTypeStr, PublicDatasetType, RampType, RecordProcessorType, SearchPlannerType, SearchRecipePostProcessType, SearchRecipeType, ServiceRunType, ServiceType, StreamExporterType, TimingMode, TransportType, UIType, URLSelectionStrategy, ZMQProxyType - from aiperf.post_processors.protocols import RecordProcessorProtocol + from aiperf.plugin.enums import APIRouterType, AccumulatorType, AccuracyBenchmarkType, AccuracyGraderType, AnalyzerType, ArrivalPattern, CommClientType, CommunicationBackend, ComposerType, ConsoleExporterType, ConvergenceCriterionType, CustomDatasetType, DataExporterType, DatasetBackingStoreType, DatasetClientStoreType, DatasetSamplingStrategy, EndpointType, GPUTelemetryCollectorType, PlotType, PluginType, PluginTypeStr, PublicDatasetType, RampType, RecordObserverType, RecordProcessorType, SearchPlannerType, SearchRecipePostProcessType, SearchRecipeType, ServiceRunType, ServiceType, StreamExporterType, TimingMode, TransportType, UIType, URLSelectionStrategy, ZMQProxyType + from aiperf.post_processors.protocols import RecordObserverProtocol, RecordProcessorProtocol from aiperf.search_recipes._base import SearchRecipe from aiperf.search_recipes.post_process import PostProcessHandler from aiperf.timing.intervals import IntervalGeneratorProtocol @@ -1006,6 +1006,10 @@ def get_class(category: Literal[PluginType.RECORD_PROCESSOR, "record_processor"] @overload def iter_all(category: Literal[PluginType.RECORD_PROCESSOR, "record_processor"]) -> Iterator[tuple[PluginEntry, type[RecordProcessorProtocol]]]: ... @overload + def get_class(category: Literal[PluginType.RECORD_OBSERVER, "record_observer"], name_or_class_path: RecordObserverType | str) -> type[RecordObserverProtocol]: ... + @overload + def iter_all(category: Literal[PluginType.RECORD_OBSERVER, "record_observer"]) -> Iterator[tuple[PluginEntry, type[RecordObserverProtocol]]]: ... + @overload def get_class(category: Literal[PluginType.ACCUMULATOR, "accumulator"], name_or_class_path: AccumulatorType | str) -> type[AccumulatorProtocol]: ... @overload def iter_all(category: Literal[PluginType.ACCUMULATOR, "accumulator"]) -> Iterator[tuple[PluginEntry, type[AccumulatorProtocol]]]: ... diff --git a/src/aiperf/plugin/plugins.yaml b/src/aiperf/plugin/plugins.yaml index 48ab29f215..c56f66d786 100644 --- a/src/aiperf/plugin/plugins.yaml +++ b/src/aiperf/plugin/plugins.yaml @@ -955,10 +955,10 @@ ramp: Provides stochastic burstiness while guaranteeing completion within duration. # ============================================================================= -# Record Processors +# Record Processors (Producers) # ============================================================================= -# Record processors stream records and compute metrics in a distributed manner. -# One-to-many mapping: multiple processors can be loaded simultaneously. +# Record PRODUCERS parse a record and emit one typed result on a declared +# record-type channel. One-to-many mapping: multiple producers loaded simultaneously. # ============================================================================= record_processor: metric_record: @@ -967,23 +967,35 @@ record_processor: Streaming record processor that computes metrics from MetricType.RECORD and MetricType.AGGREGATE. First stage of distributed processing pipeline. Always loaded. - - raw_record_writer: - class: aiperf.post_processors.raw_record_writer_processor:RawRecordWriterProcessor - description: | - Raw record writer that streams raw request/response data to JSONL files - for detailed analysis and debugging. Enabled when export_level is RAW. + metadata: + record_type: metric_records accuracy_record: class: aiperf.accuracy.accuracy_record_processor:AccuracyRecordProcessor description: | Accuracy record processor that grades LLM responses against ground truth for accuracy benchmarking. Self-disables when accuracy mode is off. + metadata: + record_type: accuracy + +# ============================================================================= +# Record Observers +# ============================================================================= +# Record OBSERVERS view the produced results + the record and act (e.g. write +# JSONL); they return nothing and emit no channel record. +# One-to-many mapping: multiple observers loaded simultaneously. +# ============================================================================= +record_observer: + raw_record_writer: + class: aiperf.post_processors.raw_record_writer_processor:RawRecordWriterProcessor + description: | + Raw record writer that streams raw request/response data to JSONL files + for detailed analysis and debugging. Enabled when export_level is RAW. outputs_json: class: aiperf.post_processors.outputs_json_record_processor:OutputsJsonRecordProcessor description: | - Outputs JSON record processor that captures model response text per request. + Outputs JSON record observer that captures model response text per request. Enabled when --export-outputs-json is set. Writes per-processor fragment files. # ============================================================================= diff --git a/src/aiperf/plugin/schema/plugins.schema.json b/src/aiperf/plugin/schema/plugins.schema.json index 8652f31eed..06ca1b2df7 100644 --- a/src/aiperf/plugin/schema/plugins.schema.json +++ b/src/aiperf/plugin/schema/plugins.schema.json @@ -109,11 +109,19 @@ "record_processor": { "title": "Record Processor Plugins", "type": "object", - "description": "Record processors stream records and compute metrics in a distributed manner.\nFirst stage of metrics pipeline, handling per-record computations.\nOne-to-many mapping: multiple processors can be loaded simultaneously.", + "description": "Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously.", "additionalProperties": { "$ref": "#/$defs/RecordProcessorPlugin" } }, + "record_observer": { + "title": "Record Observer Plugins", + "type": "object", + "description": "Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously.", + "additionalProperties": { + "$ref": "#/$defs/RecordObserverPlugin" + } + }, "accumulator": { "title": "Accumulator Plugins", "type": "object", @@ -1051,6 +1059,48 @@ "description": "Transports handle the network layer for sending requests to inference servers.\nManages connection pooling, streaming, error handling, and TCP configuration.\nOne-to-one mapping based on transport_type configuration." }, "RecordProcessorPlugin": { + "type": "object", + "properties": { + "class": { + "description": "Python class that implements this plugin entry. Use 'module.path:ClassName' format, e.g., 'aiperf.endpoints.openai_chat:ChatEndpoint'.", + "title": "Class", + "type": "string" + }, + "description": { + "default": "", + "description": "Brief explanation of what this plugin type does and when to use it.", + "title": "Description", + "type": "string" + }, + "priority": { + "default": 0, + "description": "Conflict resolution priority. When multiple packages register the same type name, the one with higher priority wins. Use 0 for normal plugins, higher values to override built-in implementations.", + "title": "Priority", + "type": "integer" + }, + "metadata": { + "description": "Metadata schema for record producer plugins.\n\nProducers parse a record and emit one typed result on a declared record-type\nchannel. RecordProcessorService groups producer outputs by this declared\n``record_type`` (rather than runtime type-sniffing) and routes each group to\nits dedicated downstream message.\n\nReferenced by: categories.yaml record_processor.metadata_class\nUsed in: plugins.yaml record_processor entries", + "properties": { + "record_type": { + "description": "The record_type channel this producer emits onto (e.g. 'metric_records', 'accuracy').", + "title": "Record Type", + "type": "string" + } + }, + "required": [ + "record_type" + ], + "title": "RecordProducerMetadata", + "type": "object" + } + }, + "required": [ + "class" + ], + "title": "Record Processor Plugin", + "description": "Record PRODUCERS parse a record and emit one typed result on a declared\nrecord-type channel. First stage of the metrics pipeline, handling per-record\ncomputations. One-to-many mapping: multiple producers can be loaded simultaneously." + }, + "RecordObserverPlugin": { "type": "object", "properties": { "class": { @@ -1088,8 +1138,8 @@ "required": [ "class" ], - "title": "Record Processor Plugin", - "description": "Record processors stream records and compute metrics in a distributed manner.\nFirst stage of metrics pipeline, handling per-record computations.\nOne-to-many mapping: multiple processors can be loaded simultaneously." + "title": "Record Observer Plugin", + "description": "Record OBSERVERS view the produced results + the record and act (e.g. write\nJSONL); they return nothing and emit no channel record. Second stage of the\nrecord-processor service. One-to-many mapping: multiple observers loaded simultaneously." }, "AccumulatorPlugin": { "type": "object", diff --git a/src/aiperf/plugin/schema/schemas.py b/src/aiperf/plugin/schema/schemas.py index 51b5dd1f86..6d38ae21a6 100644 --- a/src/aiperf/plugin/schema/schemas.py +++ b/src/aiperf/plugin/schema/schemas.py @@ -512,6 +512,23 @@ class RecordRoutingMetadata(BaseModel): ) +class RecordProducerMetadata(BaseModel): + """Metadata schema for record producer plugins. + + Producers parse a record and emit one typed result on a declared record-type + channel. RecordProcessorService groups producer outputs by this declared + ``record_type`` (rather than runtime type-sniffing) and routes each group to + its dedicated downstream message. + + Referenced by: categories.yaml record_processor.metadata_class + Used in: plugins.yaml record_processor entries + """ + + record_type: str = Field( + description="The record_type channel this producer emits onto (e.g. 'metric_records', 'accuracy').", + ) + + class AnalyzerMetadata(BaseModel): """Metadata schema for analyzer plugins. diff --git a/src/aiperf/post_processors/metric_record_processor.py b/src/aiperf/post_processors/metric_record_processor.py index be987f37c6..6587c18ceb 100644 --- a/src/aiperf/post_processors/metric_record_processor.py +++ b/src/aiperf/post_processors/metric_record_processor.py @@ -8,6 +8,7 @@ from aiperf.common.enums import MetricType from aiperf.common.exceptions import NoMetricValue +from aiperf.common.messages import MetricRecordsData from aiperf.common.models import ParsedResponseRecord from aiperf.common.models.record_models import MetricRecordMetadata from aiperf.common.types import MetricTagT @@ -56,8 +57,14 @@ def __init__( async def process_record( self, record: ParsedResponseRecord, metadata: MetricRecordMetadata - ) -> MetricRecordDict: - """Process a response record from the inference results parser.""" + ) -> MetricRecordsData: + """Process a response record from the inference results parser. + + Emits a finished ``MetricRecordsData`` (the ``metric_records`` typed + channel record) carrying the computed metrics plus the trace_data and + error captured off the record, so it ships generically alongside every + other producer's typed record. + """ record_metrics: MetricRecordDict = MetricRecordDict() parse_funcs = self.valid_parse_funcs if record.valid else self.error_parse_funcs # NOTE: Need to parse the record in a loop, as the parse_record function may depend on the results of previous metrics. @@ -70,4 +77,9 @@ async def process_record( ) except Exception as e: self.warning(f"Error parsing record for metric '{tag}': {e!r}") - return record_metrics + return MetricRecordsData( + metadata=metadata, + metrics=record_metrics, + trace_data=record.request.trace_data, + error=record.request.error, + ) diff --git a/src/aiperf/post_processors/outputs_json_record_processor.py b/src/aiperf/post_processors/outputs_json_record_processor.py index 0d984e1975..2b6055c4d1 100644 --- a/src/aiperf/post_processors/outputs_json_record_processor.py +++ b/src/aiperf/post_processors/outputs_json_record_processor.py @@ -1,6 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Record processor that captures model response text per request for outputs.json export.""" +"""Record observer that captures model response text per request for outputs.json export.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING from pydantic import Field @@ -8,11 +12,13 @@ from aiperf.common.environment import Environment from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.mixins import BufferedJSONLWriterMixin -from aiperf.common.models import MetricRecordMetadata, ParsedResponseRecord from aiperf.common.models.base_models import AIPerfBaseModel from aiperf.config.artifacts import OutputDefaults from aiperf.config.resolution.plan import BenchmarkRun +if TYPE_CHECKING: + from aiperf.post_processors.record_observer_context import RecordObserverContext + class OutputFragment(AIPerfBaseModel): """A single output fragment capturing response text and request identifiers.""" @@ -80,10 +86,10 @@ def __init__( self.info(f"OutputsJsonRecordProcessor initialized: {self.output_file}") - async def process_record( - self, record: ParsedResponseRecord, metadata: MetricRecordMetadata - ) -> None: + async def observe(self, ctx: RecordObserverContext) -> None: """Extract response text and write an output fragment.""" + record = ctx.record + metadata = ctx.metadata if metadata.benchmark_phase != CreditPhase.PROFILING: return diff --git a/src/aiperf/post_processors/protocols.py b/src/aiperf/post_processors/protocols.py index 66f79c20e5..098d04083d 100644 --- a/src/aiperf/post_processors/protocols.py +++ b/src/aiperf/post_processors/protocols.py @@ -3,20 +3,29 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable from aiperf.common.models import ParsedResponseRecord from aiperf.common.protocols import AIPerfLifecycleProtocol if TYPE_CHECKING: from aiperf.common.models.record_models import MetricRecordMetadata - from aiperf.metrics.metric_dicts import MetricRecordDict + from aiperf.post_processors.record_observer_context import RecordObserverContext @runtime_checkable class RecordProcessorProtocol(AIPerfLifecycleProtocol, Protocol): - """Protocol for per-worker record processors that extract metric records.""" + """Protocol for record PRODUCERS: parse a record and emit one typed result + on a declared record-type channel.""" async def process_record( self, record: ParsedResponseRecord, metadata: MetricRecordMetadata - ) -> MetricRecordDict: ... + ) -> Any: ... + + +@runtime_checkable +class RecordObserverProtocol(AIPerfLifecycleProtocol, Protocol): + """Protocol for record OBSERVERS: view the produced results + the record and + act (e.g. write JSONL). Observers return nothing and emit no channel record.""" + + async def observe(self, ctx: RecordObserverContext) -> None: ... diff --git a/src/aiperf/post_processors/raw_record_writer_processor.py b/src/aiperf/post_processors/raw_record_writer_processor.py index c6a9ed452c..7325c64ab5 100644 --- a/src/aiperf/post_processors/raw_record_writer_processor.py +++ b/src/aiperf/post_processors/raw_record_writer_processor.py @@ -26,6 +26,7 @@ if TYPE_CHECKING: from aiperf.config.resolution.plan import BenchmarkRun + from aiperf.post_processors.record_observer_context import RecordObserverContext class RawRecordWriterProcessor(BufferedJSONLWriterMixin[RawRecordInfo]): @@ -161,12 +162,10 @@ async def buffered_write(self, record: RawRecordInfo) -> None: self.error(f"Failed to write raw record: {e!r}") self.dropped_record_count += 1 - async def process_record( - self, record: ParsedResponseRecord, metadata: MetricRecordMetadata - ) -> None: - """Process a single record.""" + async def observe(self, ctx: RecordObserverContext) -> None: + """Write the raw request/response data for a single record.""" # Build export record with full parsed record - record_export = self._build_export_record(record, metadata) + record_export = self._build_export_record(ctx.record, ctx.metadata) # Write using the buffered writer mixin (handles batching and flushing) await self.buffered_write(record_export) diff --git a/src/aiperf/post_processors/record_observer_context.py b/src/aiperf/post_processors/record_observer_context.py new file mode 100644 index 0000000000..8223798aed --- /dev/null +++ b/src/aiperf/post_processors/record_observer_context.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Read-only context object handed to record observers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from aiperf.common.messages.inference_messages import MetricRecordsData + +if TYPE_CHECKING: + from aiperf.common.models import MetricRecordMetadata, ParsedResponseRecord + + +@dataclass(slots=True) +class RecordObserverContext: + """What an observer sees: the original record + all producer outputs for it.""" + + record: ParsedResponseRecord + """The original parsed record.""" + + metadata: MetricRecordMetadata + """The metric record metadata for this record.""" + + produced: dict[str, list[Any]] + """Producer outputs keyed by declared record_type. + + e.g. ``{"metric_records": [MetricRecordDict], "accuracy": [AccuracyRecordsData]}``. + """ + + def get(self, record_type: str) -> list[Any]: + """Return the producer outputs emitted on ``record_type`` (empty if none).""" + return self.produced.get(record_type, []) + + @property + def metrics(self) -> MetricRecordsData | None: + """The first metric-records output, or None when no producer emitted one.""" + items = self.produced.get(MetricRecordsData.record_type) or [] + return items[0] if items else None diff --git a/src/aiperf/records/record_processor_service.py b/src/aiperf/records/record_processor_service.py index 4e1dbedeed..ad1fc4bfa7 100644 --- a/src/aiperf/records/record_processor_service.py +++ b/src/aiperf/records/record_processor_service.py @@ -1,10 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import asyncio -import traceback from typing import TYPE_CHECKING, Any -from aiperf.accuracy.models import AccuracyRecordsData from aiperf.common.base_component_service import BaseComponentService from aiperf.common.enums import ( CommAddress, @@ -17,13 +15,13 @@ from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.hooks import on_command, on_message, on_pull_message from aiperf.common.messages import ( - AccuracyRecordsMessage, DatasetConfiguredNotification, InferenceResultsMessage, - MetricRecordsMessage, ProfileCompleteCommand, ProfileConfigureCommand, + RecordsMessage, ) +from aiperf.common.messages.inference_messages import MetricRecordsData from aiperf.common.mixins import PullClientMixin from aiperf.common.models import ( MetricRecordMetadata, @@ -36,10 +34,13 @@ from aiperf.common.protocols import PushClientProtocol from aiperf.common.tokenizer import Tokenizer from aiperf.common.utils import compute_time_ns -from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.plugin import plugins from aiperf.plugin.enums import PluginType -from aiperf.post_processors.protocols import RecordProcessorProtocol +from aiperf.post_processors.protocols import ( + RecordObserverProtocol, + RecordProcessorProtocol, +) +from aiperf.post_processors.record_observer_context import RecordObserverContext from aiperf.records.dataset_gate import await_dataset_configured from aiperf.records.inference_result_parser import InferenceResultParser @@ -83,36 +84,68 @@ def __init__( # any record is graded. self._dataset_configured_event: asyncio.Event = asyncio.Event() - self.records_processors: list[RecordProcessorProtocol] = [] + # Stage 1 - PRODUCERS: parse a record and emit one typed result on the + # record_type channel declared in plugins.yaml metadata. Grouped by that + # declared channel (no runtime type-sniffing). + self._producers: list[tuple[str, RecordProcessorProtocol]] = [] for entry in plugins.iter_entries(PluginType.RECORD_PROCESSOR): try: - ProcessorClass = plugins.get_class( + ProducerClass = plugins.get_class( PluginType.RECORD_PROCESSOR, entry.name ) - processor: RecordProcessorProtocol = ProcessorClass( + producer: RecordProcessorProtocol = ProducerClass( + run=self.run, + service_id=self.service_id, + ) + record_type = entry.metadata["record_type"] + self._producers.append((record_type, producer)) + self.attach_child_lifecycle(producer) + self.debug( + f"Created record producer: {entry.name} ({record_type}): {producer.__class__.__name__}" + ) + except PostProcessorDisabled: + self.debug( + f"Record producer {entry.name} is disabled and will not be used" + ) + except Exception as e: + self.exception(f"Error creating record producer: {e!r}") + raise + + # Stage 2 - OBSERVERS: view the produced results + the record and act + # (e.g. write JSONL). They return nothing and emit no channel record. + self._observers: list[RecordObserverProtocol] = [] + for entry in plugins.iter_entries(PluginType.RECORD_OBSERVER): + try: + ObserverClass = plugins.get_class( + PluginType.RECORD_OBSERVER, entry.name + ) + observer: RecordObserverProtocol = ObserverClass( run=self.run, service_id=self.service_id, ) - self.records_processors.append(processor) - self.attach_child_lifecycle(processor) + self._observers.append(observer) + self.attach_child_lifecycle(observer) self.debug( - f"Created record processor: {entry.name}: {processor.__class__.__name__}" + f"Created record observer: {entry.name}: {observer.__class__.__name__}" ) except PostProcessorDisabled: self.debug( - f"Record processor {entry.name} is disabled and will not be used" + f"Record observer {entry.name} is disabled and will not be used" ) except Exception as e: - self.exception(f"Error creating record processor: {e!r}") + self.exception(f"Error creating record observer: {e!r}") raise @on_message(MessageType.DATASET_CONFIGURED_NOTIFICATION) async def _on_dataset_configured( self, message: DatasetConfiguredNotification ) -> None: - for processor in self.records_processors: - if hasattr(processor, "on_dataset_configured"): - processor.on_dataset_configured(message.metadata) + for _record_type, producer in self._producers: + if hasattr(producer, "on_dataset_configured"): + producer.on_dataset_configured(message.metadata) + for observer in self._observers: + if hasattr(observer, "on_dataset_configured"): + observer.on_dataset_configured(message.metadata) self._dataset_configured_event.set() @on_command(CommandType.PROFILE_CONFIGURE) @@ -215,7 +248,7 @@ async def _on_inference_results(self, message: InferenceResultsMessage) -> None: """Handle an inference results message. Lockstep contract: every received message forwards exactly one - ``MetricRecordsMessage``. The worker has already returned the credit as + ``RecordsMessage``. The worker has already returned the credit as completed by the time the record arrives here, so a dropped record leaves the RecordsManager completion barrier (``success_records + error_records >= final_requests_completed``, which has no timeout) @@ -263,7 +296,7 @@ async def _process_and_forward_record( record: RequestRecord, last_response_perf_ns: int | None, ) -> None: - """Parse, process, and forward the metric record for a single request.""" + """Parse, produce, observe, and forward the records for a single request.""" parsed_record = await self.inference_result_parser.parse_request_record(record) # Free raw SSE messages now that parsing extracted what it needs. @@ -275,68 +308,63 @@ async def _process_and_forward_record( record, message.service_id, last_response_perf_ns ) - raw_results = await self._process_record(parsed_record, metadata) - - trace_data, error = self._free_record_data(record, parsed_record) - - results = [] - for result in raw_results: + # Stage 1 - producers: run concurrently, group outputs by declared channel. + by_type: dict[str, list[Any]] = {} + producer_results = await asyncio.gather( + *[ + producer.process_record(parsed_record, metadata) + for _record_type, producer in self._producers + ], + return_exceptions=True, + ) + for (record_type, _producer), result in zip( + self._producers, producer_results, strict=True + ): + if isinstance(result, BaseException): + self.error(f"Error in producer for {record_type}: {result!r}") + continue + if result is None: + continue + by_type.setdefault(record_type, []).append(result) + + # Stage 2 - observers: view the produced results + the record and act. + # Must run BEFORE _free_record_data so they can read the full parsed + # record via ctx.record. + ctx = RecordObserverContext( + record=parsed_record, + metadata=metadata, + produced=by_type, + ) + observer_results = await asyncio.gather( + *[observer.observe(ctx) for observer in self._observers], + return_exceptions=True, + ) + for observer, result in zip(self._observers, observer_results, strict=True): if isinstance(result, BaseException): self.error( - f"Error processing record: {result!r}: {traceback.format_exception(result)}" + f"Error in observer {observer.__class__.__name__}: {result!r}" ) - else: - results.append(result) - - # Partition processor outputs by transport: MetricRecordDict is a dict - # subclass and merges into MetricRecordsMessage.results; typed Pydantic - # record models (e.g. AccuracyRecordsData) travel on dedicated channels. - # A typed record leaking into results would break to_data()'s dict-merge. - metric_dicts = [r for r in results if isinstance(r, dict)] - typed_records = [r for r in results if not isinstance(r, dict)] + _trace_data, error = self._free_record_data(record, parsed_record) + + # Ship generically: ONE RecordsMessage per inference record carries the + # request envelope (metadata + request-level error) plus every produced + # typed record flattened into one list. Each record self-identifies via + # its own record_type ClassVar, so no per-type message class or builder + # map is needed. Always pushed (even when no producer emitted a record) + # to keep the RecordsManager completion barrier in lockstep with the + # credit. The metric producer already put trace_data inside its + # MetricRecordsData, so trace_data is not carried on the envelope. + all_records = [record for records in by_type.values() for record in records] await self.records_push_client.push( - MetricRecordsMessage( + RecordsMessage( service_id=self.service_id, metadata=metadata, - results=metric_dicts, - trace_data=trace_data, + records=all_records, error=error, ) ) - await self._push_typed_records(typed_records) - - async def _push_typed_records( - self, typed_records: list[AccuracyRecordsData] - ) -> None: - """Push typed record models on their dedicated channels. - - Groups records by their ``record_type`` classvar and builds one dedicated - message per type via a generic ``record_type -> message builder`` mapping, - so future typed records reuse this seam instead of adding an if-branch. - """ - if not typed_records: - return - - builders = { - AccuracyRecordsData.record_type: lambda records: AccuracyRecordsMessage( - service_id=self.service_id, - records=records, - ), - } - - grouped: dict[str, list[Any]] = {} - for record in typed_records: - grouped.setdefault(record.record_type, []).append(record) - - for record_type, records in grouped.items(): - builder = builders.get(record_type) - if builder is None: - self.error(f"No message builder for typed record type: {record_type}") - continue - await self.records_push_client.push(builder(records)) - async def _forward_failed_record( self, message: InferenceResultsMessage, @@ -371,13 +399,16 @@ async def _forward_failed_record( record_processor_id=self.service_id, benchmark_phase=benchmark_phase, ) + error = record.error or ErrorDetails.from_exception(exc) + # The producers didn't run, so ship a RecordsMessage carrying a single + # errored MetricRecordsData (empty metrics) so the accumulator still sees + # the record and the records-tracker lockstep counts it. await self.records_push_client.push( - MetricRecordsMessage( + RecordsMessage( service_id=self.service_id, metadata=metadata, - results=[], - trace_data=None, - error=record.error or ErrorDetails.from_exception(exc), + records=[MetricRecordsData(metadata=metadata, metrics={}, error=error)], + error=error, ) ) @@ -387,8 +418,8 @@ def _free_record_data( """Free large data structures from the record after all processors have run. All metrics and post-processors consume these fields during _process_record(). - The only data sent downstream in MetricRecordsMessage is metadata, results, - trace_data, and error -- so everything else can be released here. + The only data sent downstream is the typed records produced for this request + (metadata, metrics, trace_data, error) -- so everything else can be released here. We assign None to fields typed as non-optional lists (turns, responses) to let the GC reclaim the underlying objects. Using .clear() would keep the empty list @@ -408,19 +439,6 @@ def _free_record_data( parsed_record.responses = None return trace_data, error - async def _process_record( - self, record: ParsedResponseRecord, metadata: MetricRecordMetadata - ) -> list[MetricRecordDict | AccuracyRecordsData | BaseException]: - """Stream a record to the records processors.""" - tasks = [ - processor.process_record(record, metadata) - for processor in self.records_processors - ] - results: list[ - MetricRecordDict | AccuracyRecordsData | BaseException | None - ] = await asyncio.gather(*tasks, return_exceptions=True) - return [result for result in results if result is not None] - def main() -> None: from aiperf.common.bootstrap import bootstrap_and_run_service diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 15f10be491..c8b6d569e9 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -26,11 +26,8 @@ from aiperf.common.environment import Environment from aiperf.common.hooks import background_task, on_command, on_message, on_pull_message from aiperf.common.messages import ( - AccuracyRecordsMessage, AllRecordsReceivedMessage, DatasetConfiguredNotification, - MetricRecordsData, - MetricRecordsMessage, NetworkLatencyRecordMessage, ProcessAccuracyResultMessage, ProcessAllResultsMessage, @@ -42,6 +39,7 @@ ProfileCompleteCommand, RealtimeMetricsCommand, RealtimeMetricsMessage, + RecordsMessage, RecordsProcessingStatsMessage, ServerMetricsRecordMessage, StartRealtimeTelemetryCommand, @@ -72,10 +70,6 @@ ) from aiperf.gpu_telemetry.protocols import GPUTelemetryAccumulatorProtocol from aiperf.metrics.accumulator_models import AccumulatorMetricsSummary -from aiperf.metrics.cache_reporting_hint import ( - CACHE_REPORTING_HINT, - usage_without_cache_in_record, -) from aiperf.network_latency.accumulator import NetworkLatencyAccumulator from aiperf.plugin import plugins from aiperf.plugin.enums import ( @@ -359,9 +353,6 @@ class RecordsManager(PullClientMixin, BaseComponentService): many records before finalizing results. """ - # The "enable cache reporting" server-knob hint fires at most once per run. - _warned_missing_cache_reporting: bool = False - def __init__( self, run: BenchmarkRun, @@ -529,25 +520,29 @@ def _log_routing_table(self) -> None: handler_names = [handler.__class__.__name__ for handler in handlers] self.debug(lambda rt=record_type, hn=handler_names: f" {rt} -> {hn}") - @on_pull_message(MessageType.METRIC_RECORDS) - async def _on_metric_records(self, message: MetricRecordsMessage) -> None: - """Handle a metric records message.""" + @on_pull_message(MessageType.RECORDS) + async def _on_records(self, message: RecordsMessage) -> None: + """Handle a per-request records envelope generically. + + One ``RecordsMessage`` == one inference request. Each contained record + self-identifies via its ``record_type`` ClassVar and is dispatched to its + registered handlers; the per-request lockstep keys off the message + envelope (``message.metadata`` / ``message.error``), never off sniffing a + record type. + """ if not await await_dataset_configured(self, self._dataset_configured_event): return if self.is_trace_enabled: - self.trace(f"Received metric records: {message}") + self.trace(f"Received records: {message}") - record_data = message.to_data() - - self._maybe_hint_missing_cache_reporting(record_data) + dispatch_errors: list[BaseException] = [] + for record in message.records: + dispatch_errors.extend(await self._dispatch_record(record)) - phase = record_data.metadata.benchmark_phase - dispatch_errors = await self._dispatch_record(record_data) - self._records_tracker.update_from_record_data(record_data) - if record_data.error: - self._error_tracker.increment_error_count_for_phase( - phase, record_data.error - ) + phase = message.metadata.benchmark_phase + self._records_tracker.update_from_request(message.metadata, message.error) + if message.error: + self._error_tracker.increment_error_count_for_phase(phase, message.error) # A metric accumulator/exporter that failed to ingest this record yields # incomplete metrics; surface it in the phase error summary rather than # marking the record cleanly processed and silently dropping the failure. @@ -589,18 +584,6 @@ async def _on_server_metrics_records( elif message.error: self._server_metrics_state.error_counts[message.error] += 1 - @on_pull_message(MessageType.ACCURACY_RECORD) - async def _on_accuracy_records(self, message: AccuracyRecordsMessage) -> None: - """Handle graded accuracy records from a record processor. - - The routing table auto-includes ``"accuracy"`` from plugin metadata, so - ``_dispatch_record`` fans each record out to the AccuracyAccumulator and - AccuracyJSONLWriter without any accuracy-specific routing code here. - """ - for record in message.records: - for error in await self._dispatch_record(record): - self.debug(lambda e=error: f"Accuracy record dispatch error: {e!r}") - @on_pull_message(MessageType.NETWORK_LATENCY_RECORD) async def _on_network_latency_records( self, message: NetworkLatencyRecordMessage @@ -1040,21 +1023,6 @@ async def _generate_realtime_metrics(self) -> list[MetricResult]: """ return await generate_realtime_metrics(self._metric_record_accumulators) - def _maybe_hint_missing_cache_reporting( - self, record_data: MetricRecordsData - ) -> None: - """Warn once, mid-run, when the server reports token usage but no prompt-cache - reads — the signature of a cache-capable server that hasn't been told to - report ``cached_tokens``. Fires on the first qualifying record so a long run - can be aborted and re-launched with the flag set; the end-of-run console - exporter emits the same hint for anyone who only reads the final summary. - """ - if self._warned_missing_cache_reporting: - return - if usage_without_cache_in_record(record_data.metrics): - self._warned_missing_cache_reporting = True - self.warning(CACHE_REPORTING_HINT) - async def _run_analyzers(self, ctx: SummaryContext) -> list[MetricResult]: """Run summarize-time analyzer plugins that join across accumulators. diff --git a/src/aiperf/records/records_tracker.py b/src/aiperf/records/records_tracker.py index c05e0e4c49..b6f79188d6 100644 --- a/src/aiperf/records/records_tracker.py +++ b/src/aiperf/records/records_tracker.py @@ -5,10 +5,11 @@ from collections import defaultdict from aiperf.common.enums import CreditPhase -from aiperf.common.messages import MetricRecordsData from aiperf.common.mixins import AIPerfLoggerMixin from aiperf.common.models import ( CreditPhaseStats, + ErrorDetails, + MetricRecordMetadata, PhaseRecordsStats, WorkerProcessingStats, ) @@ -190,17 +191,21 @@ def update_phase_info(self, credit_phase_stats: CreditPhaseStats) -> None: phase_tracker = self._get_phase_tracker(credit_phase_stats.phase) phase_tracker.update_from_credit_phase_stats(credit_phase_stats) - def update_from_record_data(self, record_data: MetricRecordsData) -> None: - """Update the phase tracker from a record data.""" - phase_tracker = self._get_phase_tracker(record_data.metadata.benchmark_phase) - if record_data.valid: + def update_from_request( + self, metadata: MetricRecordMetadata, error: ErrorDetails | None + ) -> None: + """Update the phase tracker for one completed request. + + Drives the per-request lockstep off the request envelope: a request is + counted as a success when ``error is None``, otherwise as an error. + """ + phase_tracker = self._get_phase_tracker(metadata.benchmark_phase) + if error is None: phase_tracker.increment_success_records() - phase_tracker.increment_worker_success_records( - record_data.metadata.worker_id - ) + phase_tracker.increment_worker_success_records(metadata.worker_id) else: phase_tracker.increment_error_records() - phase_tracker.increment_worker_error_records(record_data.metadata.worker_id) + phase_tracker.increment_worker_error_records(metadata.worker_id) def was_phase_cancelled(self, phase: CreditPhase) -> bool: """Check if the phase was cancelled.""" diff --git a/tests/unit/accuracy/test_accuracy_models.py b/tests/unit/accuracy/test_accuracy_models.py index 50fced61c5..8ea9b650fa 100644 --- a/tests/unit/accuracy/test_accuracy_models.py +++ b/tests/unit/accuracy/test_accuracy_models.py @@ -3,9 +3,6 @@ from __future__ import annotations -import pytest -from pytest import param - from aiperf.accuracy.models import ( AccuracyRecordsData, AccuracySummary, @@ -14,9 +11,10 @@ ) from aiperf.common.enums import CreditPhase, MessageType from aiperf.common.messages import ( - AccuracyRecordsMessage, ProcessAccuracyResultMessage, + RecordsMessage, ) +from aiperf.common.models.record_models import MetricRecordMetadata def _make_record(**overrides) -> AccuracyRecordsData: @@ -99,27 +97,30 @@ def test_process_accuracy_result_defaults_none() -> None: assert wrapped.results.total_evaluated == 5 -@pytest.mark.parametrize( - "message_cls, expected_type", - [ - param(AccuracyRecordsMessage, MessageType.ACCURACY_RECORD, id="records"), - param( - ProcessAccuracyResultMessage, - MessageType.PROCESS_ACCURACY_RESULT, - id="result", - ), - ], -) # fmt: skip -def test_accuracy_message_types(message_cls, expected_type) -> None: - assert message_cls.model_fields["message_type"].default == expected_type +def test_process_accuracy_result_message_type() -> None: + assert ( + ProcessAccuracyResultMessage.model_fields["message_type"].default + == MessageType.PROCESS_ACCURACY_RESULT + ) -def test_accuracy_records_message_serializes_records() -> None: - message = AccuracyRecordsMessage( +def test_records_message_serializes_accuracy_records() -> None: + """Accuracy records ride the generic RecordsMessage envelope and serialize + with their own record_type-carrying fields intact.""" + metadata = MetricRecordMetadata( + session_num=0, + request_start_ns=1_000, + request_end_ns=2_000, + worker_id="worker-1", + record_processor_id="rp", + benchmark_phase=CreditPhase.PROFILING, + ) + message = RecordsMessage( service_id="records-manager", + metadata=metadata, records=[_make_record(session_num=1), _make_record(session_num=2)], ) - assert message.message_type == MessageType.ACCURACY_RECORD + assert message.message_type == MessageType.RECORDS dumped = message.model_dump() assert len(dumped["records"]) == 2 assert dumped["records"][0]["session_num"] == 1 diff --git a/tests/unit/post_processors/conftest.py b/tests/unit/post_processors/conftest.py index 34655ba2e5..52f59fd3af 100644 --- a/tests/unit/post_processors/conftest.py +++ b/tests/unit/post_processors/conftest.py @@ -15,14 +15,13 @@ from aiperf.common.enums import ( CreditPhase, ExportLevel, - MessageType, MetricFlags, MetricValueTypeT, ModelSelectionStrategy, ) from aiperf.common.enums.metric_enums import GenericMetricUnit from aiperf.common.exceptions import NoMetricValue -from aiperf.common.messages import MetricRecordsMessage +from aiperf.common.messages import MetricRecordsData from aiperf.common.mixins import AIPerfLifecycleMixin from aiperf.common.models import ( ErrorDetails, @@ -794,7 +793,7 @@ def create_metric_metadata( ) -def create_metric_records_message( +def create_metric_records_data( service_id: str = "test-processor", results: list[dict[MetricTagT, MetricValueTypeT]] | None = None, error: ErrorDetails | None = None, @@ -802,13 +801,13 @@ def create_metric_records_message( x_request_id: str | None = None, trace_data: Any | None = None, **metadata_kwargs, -) -> MetricRecordsMessage: +) -> MetricRecordsData: """ - Create a MetricRecordsMessage with sensible defaults. + Create a finished MetricRecordsData with sensible defaults. Args: - service_id: Service ID - results: List of metric result dictionaries + service_id: Service ID (unused; kept for call-site compatibility) + results: List of metric result dictionaries, merged into the metrics dict error: Error details if any metadata: Pre-built metadata, or None to build from kwargs x_request_id: Record ID (set as x_request_id in metadata if provided) @@ -816,7 +815,7 @@ def create_metric_records_message( **metadata_kwargs: Args passed to create_metric_metadata if metadata is None Returns: - MetricRecordsMessage object + MetricRecordsData object """ if results is None: results = [] @@ -827,13 +826,15 @@ def create_metric_records_message( metadata_kwargs["x_request_id"] = x_request_id metadata = create_metric_metadata(**metadata_kwargs) - return MetricRecordsMessage( - message_type=MessageType.METRIC_RECORDS, - service_id=service_id, + metrics: dict[MetricTagT, MetricValueTypeT] = {} + for result in results: + metrics.update(result) + + return MetricRecordsData( metadata=metadata, - results=results, - error=error, + metrics=metrics, trace_data=trace_data, + error=error, ) diff --git a/tests/unit/post_processors/test_metric_record_processor.py b/tests/unit/post_processors/test_metric_record_processor.py index e4598a5870..78a6bc292c 100644 --- a/tests/unit/post_processors/test_metric_record_processor.py +++ b/tests/unit/post_processors/test_metric_record_processor.py @@ -6,8 +6,8 @@ import pytest from aiperf.common.exceptions import NoMetricValue +from aiperf.common.messages import MetricRecordsData from aiperf.common.models import ParsedResponseRecord -from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.metrics.types.error_request_count import ErrorRequestCountMetric from aiperf.metrics.types.request_count_metric import RequestCountMetric from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric @@ -69,11 +69,11 @@ async def test_process_valid_record( metadata = create_metric_metadata() result = await processor.process_record(sample_parsed_record, metadata) - assert isinstance(result, MetricRecordDict) - assert RequestLatencyMetric.tag in result + assert isinstance(result, MetricRecordsData) + assert RequestLatencyMetric.tag in result.metrics expected_latency = DEFAULT_LAST_RESPONSE_NS - DEFAULT_START_TIME_NS - assert result[RequestLatencyMetric.tag] == expected_latency + assert result.metrics[RequestLatencyMetric.tag] == expected_latency @pytest.mark.asyncio async def test_process_error_record( @@ -91,8 +91,8 @@ async def test_process_error_record( metadata = create_metric_metadata() result = await processor.process_record(error_parsed_record, metadata) - assert isinstance(result, MetricRecordDict) - assert result[ErrorRequestCountMetric.tag] == 1 + assert isinstance(result, MetricRecordsData) + assert result.metrics[ErrorRequestCountMetric.tag] == 1 @pytest.mark.asyncio async def test_process_record_multiple_metrics( @@ -110,13 +110,13 @@ async def test_process_record_multiple_metrics( metadata = create_metric_metadata() result = await processor.process_record(sample_parsed_record, metadata) - assert len(result) == 2 - assert RequestLatencyMetric.tag in result - assert RequestCountMetric.tag in result - assert result[RequestCountMetric.tag] == 1 + assert len(result.metrics) == 2 + assert RequestLatencyMetric.tag in result.metrics + assert RequestCountMetric.tag in result.metrics + assert result.metrics[RequestCountMetric.tag] == 1 expected_latency = DEFAULT_LAST_RESPONSE_NS - DEFAULT_START_TIME_NS - assert result[RequestLatencyMetric.tag] == expected_latency + assert result.metrics[RequestLatencyMetric.tag] == expected_latency @pytest.mark.asyncio async def test_process_record_handles_no_metric_value_exception( @@ -137,8 +137,8 @@ async def test_process_record_handles_no_metric_value_exception( with patch.object(processor, "trace") as mock_trace: result = await processor.process_record(sample_parsed_record, metadata) - assert isinstance(result, MetricRecordDict) - assert failing_metric_no_value_cls.tag not in result + assert isinstance(result, MetricRecordsData) + assert failing_metric_no_value_cls.tag not in result.metrics mock_trace.assert_called_once() assert ( f"No metric value for metric '{failing_metric_no_value_cls.tag}'" @@ -169,8 +169,8 @@ async def test_process_record_handles_value_error_exception( with patch.object(processor, "warning") as mock_warning: result = await processor.process_record(sample_parsed_record, metadata) - assert isinstance(result, MetricRecordDict) - assert failing_metric_value_error_cls.tag not in result + assert isinstance(result, MetricRecordsData) + assert failing_metric_value_error_cls.tag not in result.metrics mock_warning.assert_called_once() assert ( f"Error parsing record for metric '{failing_metric_value_error_cls.tag}'" @@ -196,12 +196,12 @@ async def test_process_record_mixed_success_failure( with patch.object(processor, "debug"): result = await processor.process_record(sample_parsed_record, metadata) - assert len(result) == 1 - assert RequestLatencyMetric.tag in result - assert failing_metric_no_value_cls.tag not in result + assert len(result.metrics) == 1 + assert RequestLatencyMetric.tag in result.metrics + assert failing_metric_no_value_cls.tag not in result.metrics expected_latency = DEFAULT_LAST_RESPONSE_NS - DEFAULT_START_TIME_NS - assert result[RequestLatencyMetric.tag] == expected_latency + assert result.metrics[RequestLatencyMetric.tag] == expected_latency @pytest.mark.asyncio async def test_process_record_with_dependencies( @@ -222,14 +222,14 @@ async def test_process_record_with_dependencies( metadata = create_metric_metadata() result = await processor.process_record(sample_parsed_record, metadata) - assert len(result) == 2 - assert RequestLatencyMetric.tag in result - assert double_latency_test_metric_cls.tag in result + assert len(result.metrics) == 2 + assert RequestLatencyMetric.tag in result.metrics + assert double_latency_test_metric_cls.tag in result.metrics # Dependent metric value is 2x the base metric value assert ( - result[double_latency_test_metric_cls.tag] - == result[RequestLatencyMetric.tag] * 2 + result.metrics[double_latency_test_metric_cls.tag] + == result.metrics[RequestLatencyMetric.tag] * 2 ) @pytest.mark.asyncio @@ -244,5 +244,5 @@ async def test_process_record_empty_metrics( metadata = create_metric_metadata() result = await processor.process_record(sample_parsed_record, metadata) - assert isinstance(result, MetricRecordDict) - assert len(result) == 0 + assert isinstance(result, MetricRecordsData) + assert len(result.metrics) == 0 diff --git a/tests/unit/post_processors/test_metric_results_strategy.py b/tests/unit/post_processors/test_metric_results_strategy.py index c3ae1b02ee..b031fd0094 100644 --- a/tests/unit/post_processors/test_metric_results_strategy.py +++ b/tests/unit/post_processors/test_metric_results_strategy.py @@ -11,7 +11,7 @@ from aiperf.common.enums import CreditPhase from aiperf.common.models import CreditPhaseStats from aiperf.post_processors.strategies.metric_results import MetricResultsStrategy -from tests.unit.post_processors.conftest import create_metric_records_message +from tests.unit.post_processors.conftest import create_metric_records_data @dataclass @@ -74,9 +74,7 @@ def test_supports_only_metric_record_data(self) -> None: strategy = MetricResultsStrategy( _MetricStrategyContext(attributes={}, coerced_values={}) ) - metric_record = create_metric_records_message( - results=[{"request_latency_ns": 1}] - ).to_data() + metric_record = create_metric_records_data(results=[{"request_latency_ns": 1}]) timing_stats = CreditPhaseStats(phase=CreditPhase.PROFILING) assert strategy.supports(metric_record) is True @@ -93,7 +91,7 @@ async def test_process_records_numeric_metric_values_into_histograms(self) -> No }, ) strategy = MetricResultsStrategy(context) - metric_record = create_metric_records_message( + metric_record = create_metric_records_data( results=[ { "request_latency": 123_000_000, @@ -101,7 +99,7 @@ async def test_process_records_numeric_metric_values_into_histograms(self) -> No "ignored_metric": [], } ] - ).to_data() + ) await strategy.process(metric_record) diff --git a/tests/unit/post_processors/test_metrics_accumulator.py b/tests/unit/post_processors/test_metrics_accumulator.py index de390ba3ee..40d1ea6f70 100644 --- a/tests/unit/post_processors/test_metrics_accumulator.py +++ b/tests/unit/post_processors/test_metrics_accumulator.py @@ -30,7 +30,7 @@ from aiperf.metrics.types.request_throughput_metric import RequestThroughputMetric from tests.unit.post_processors.conftest import ( create_accumulator_with_metrics, - create_metric_records_message, + create_metric_records_data, ) @@ -54,25 +54,25 @@ async def test_process_record_record_metric( processor = MetricsAccumulator(mock_run) processor._tags_to_types = {"test_record": MetricType.RECORD} - message = create_metric_records_message( + message = create_metric_records_data( x_request_id="test-1", session_num=0, results=[{"test_record": 42.0}], ) - await processor.process_record(message.to_data()) + await processor.process_record(message) assert "test_record" in processor._column_store.numeric_tags() values = processor._column_store.numeric("test_record") assert list(values[~np.isnan(values)]) == [42.0] # New data should expand the column store - message2 = create_metric_records_message( + message2 = create_metric_records_data( x_request_id="test-2", session_num=1, request_start_ns=1_000_000_001, results=[{"test_record": 84.0}], ) - await processor.process_record(message2.to_data()) + await processor.process_record(message2) values = processor._column_store.numeric("test_record") assert list(values[~np.isnan(values)]) == [42.0, 84.0] @@ -84,12 +84,12 @@ async def test_process_record_record_metric_list_values( processor = MetricsAccumulator(mock_run) processor._tags_to_types = {"test_record": MetricType.RECORD} - message = create_metric_records_message( + message = create_metric_records_data( x_request_id="test-1", session_num=0, results=[{"test_record": [10.0, 20.0, 30.0]}], ) - await processor.process_record(message.to_data()) + await processor.process_record(message) assert "test_record" in processor._column_store.ragged_tags() ragged = processor._column_store.ragged("test_record") @@ -106,24 +106,24 @@ async def test_process_record_aggregate_metric( RequestCountMetric.tag: AggregationKind.SUM, } - message1 = create_metric_records_message( + message1 = create_metric_records_data( x_request_id="test-1", session_num=0, results=[{RequestCountMetric.tag: 5}], ) - await processor.process_record(message1.to_data()) + await processor.process_record(message1) assert RequestCountMetric.tag in processor._column_store.numeric_tags() values = processor._column_store.numeric(RequestCountMetric.tag) assert list(values[~np.isnan(values)]) == [5.0] - message2 = create_metric_records_message( + message2 = create_metric_records_data( x_request_id="test-2", session_num=1, request_start_ns=1_000_000_001, results=[{RequestCountMetric.tag: 3}], ) - await processor.process_record(message2.to_data()) + await processor.process_record(message2) values = processor._column_store.numeric(RequestCountMetric.tag) assert list(values[~np.isnan(values)]) == [5.0, 3.0] @@ -140,13 +140,13 @@ async def test_aggregate_sum_computed_at_summary_time( processor._metric_classes = {RequestCountMetric.tag: RequestCountMetric} for i in range(3): - msg = create_metric_records_message( + msg = create_metric_records_data( x_request_id=f"test-{i}", session_num=i, request_start_ns=1_000_000_000 + i, results=[{RequestCountMetric.tag: 5}], ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) results = processor._compute_results() assert results[RequestCountMetric.tag].avg == 15.0 @@ -157,13 +157,13 @@ async def test_record_count(self, mock_metric_registry: Mock, mock_run) -> None: processor = MetricsAccumulator(mock_run) processor._tags_to_types = {} - msg1 = create_metric_records_message(x_request_id="test-1", session_num=0) - msg2 = create_metric_records_message( + msg1 = create_metric_records_data(x_request_id="test-1", session_num=0) + msg2 = create_metric_records_data( x_request_id="test-2", session_num=1, request_start_ns=1_000_000_001 ) - await processor.process_record(msg1.to_data()) - await processor.process_record(msg2.to_data()) + await processor.process_record(msg1) + await processor.process_record(msg2) assert processor.record_count == 2 @@ -176,14 +176,14 @@ async def test_export_results_separates_warmup_and_profiling_with_reused_session processor._tags_to_types = {RequestLatencyMetric.tag: MetricType.RECORD} processor._metric_classes = {RequestLatencyMetric.tag: RequestLatencyMetric} - warmup_msg = create_metric_records_message( + warmup_msg = create_metric_records_data( session_num=0, benchmark_phase=CreditPhase.WARMUP, request_start_ns=1_000_000_000, request_end_ns=1_100_000_000, results=[{RequestLatencyMetric.tag: 100_000_000.0}], ) - profiling_msg = create_metric_records_message( + profiling_msg = create_metric_records_data( session_num=0, benchmark_phase=CreditPhase.PROFILING, request_start_ns=2_000_000_000, @@ -191,8 +191,8 @@ async def test_export_results_separates_warmup_and_profiling_with_reused_session results=[{RequestLatencyMetric.tag: 200_000_000.0}], ) - await processor.process_record(warmup_msg.to_data()) - await processor.process_record(profiling_msg.to_data()) + await processor.process_record(warmup_msg) + await processor.process_record(profiling_msg) assert processor.record_count == 2 @@ -222,7 +222,7 @@ async def test_summarize_realtime_phase_scoped_excludes_warmup( } processor._aggregation_kinds = {RequestCountMetric.tag: AggregationKind.SUM} - warmup_msg = create_metric_records_message( + warmup_msg = create_metric_records_data( session_num=0, benchmark_phase=CreditPhase.WARMUP, request_start_ns=1_000_000_000, @@ -234,7 +234,7 @@ async def test_summarize_realtime_phase_scoped_excludes_warmup( } ], ) - profiling_msg = create_metric_records_message( + profiling_msg = create_metric_records_data( session_num=0, benchmark_phase=CreditPhase.PROFILING, request_start_ns=2_000_000_000, @@ -247,8 +247,8 @@ async def test_summarize_realtime_phase_scoped_excludes_warmup( ], ) - await processor.process_record(warmup_msg.to_data()) - await processor.process_record(profiling_msg.to_data()) + await processor.process_record(warmup_msg) + await processor.process_record(profiling_msg) assert processor.record_count == 2 @@ -264,13 +264,13 @@ async def test_process_record_metric_accumulates_values(self, mock_run) -> None: accumulator = MetricsAccumulator(mock_run) for idx, value in enumerate([42_000_000.0, 84_000_000.0]): - message = create_metric_records_message( + message = create_metric_records_data( x_request_id=f"test-{idx}", request_start_ns=1_000_000_000 + idx, request_end_ns=1_100_000_000 + idx, results=[{RequestLatencyMetric.tag: value}], ) - await accumulator.process_record(message.to_data()) + await accumulator.process_record(message) summary = await accumulator.summarize() result = summary.results[RequestLatencyMetric.tag] @@ -286,7 +286,7 @@ async def test_process_record_list_metric_summarizes_distribution( self, mock_run ) -> None: accumulator = MetricsAccumulator(mock_run) - message = create_metric_records_message( + message = create_metric_records_data( x_request_id="test-1", results=[ { @@ -299,7 +299,7 @@ async def test_process_record_list_metric_summarizes_distribution( ], ) - await accumulator.process_record(message.to_data()) + await accumulator.process_record(message) summary = await accumulator.summarize() result = summary.results[InterChunkLatencyMetric.tag] @@ -313,12 +313,12 @@ async def test_process_aggregate_metric_sums_values(self, mock_run) -> None: accumulator = MetricsAccumulator(mock_run) for idx, value in enumerate([5, 3]): - message = create_metric_records_message( + message = create_metric_records_data( x_request_id=f"test-{idx}", request_start_ns=1_000_000_000 + idx, results=[{RequestCountMetric.tag: value}], ) - await accumulator.process_record(message.to_data()) + await accumulator.process_record(message) summary = await accumulator.summarize() @@ -329,7 +329,7 @@ async def test_derived_metrics_are_computed_from_accumulated_results( self, mock_run ) -> None: accumulator = MetricsAccumulator(mock_run) - message = create_metric_records_message( + message = create_metric_records_data( x_request_id="test-1", results=[ { @@ -340,7 +340,7 @@ async def test_derived_metrics_are_computed_from_accumulated_results( ], ) - await accumulator.process_record(message.to_data()) + await accumulator.process_record(message) summary = await accumulator.summarize() assert summary.results[RequestThroughputMetric.tag].avg == pytest.approx(10.0) @@ -382,10 +382,10 @@ def failing_derive_func(results_dict: MetricResultsDict) -> float: async def test_summarize_returns_typed_summary(self, mock_run) -> None: accumulator = MetricsAccumulator(mock_run) await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id="test-1", results=[{RequestLatencyMetric.tag: 42_000_000.0}], - ).to_data() + ) ) summary = await accumulator.summarize() @@ -419,12 +419,12 @@ def spy_derive(results_dict: MetricResultsDict) -> float: processor._derive_funcs = {RequestThroughputMetric.tag: spy_derive} processor._metric_classes[RequestThroughputMetric.tag] = RequestThroughputMetric - msg = create_metric_records_message( + msg = create_metric_records_data( x_request_id="test-1", session_num=0, results=[{RequestCountMetric.tag: 10}], ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) processor._compute_results( window_start_ns=1_000_000_000, window_end_ns=5_000_000_000 @@ -455,12 +455,12 @@ def spy_derive(results_dict: MetricResultsDict) -> float: processor._derive_funcs = {RequestThroughputMetric.tag: spy_derive} processor._metric_classes[RequestThroughputMetric.tag] = RequestThroughputMetric - msg = create_metric_records_message( + msg = create_metric_records_data( x_request_id="test-1", session_num=0, results=[{RequestCountMetric.tag: 10}], ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) mask = np.ones(processor._column_store.count, dtype=bool) processor.compute_results_for_mask( @@ -516,9 +516,9 @@ async def test_single_record_inside( ) -> None: processor = MetricsAccumulator(mock_run) processor._tags_to_types = {} - record = create_metric_records_message( + record = create_metric_records_data( x_request_id="test-1", session_num=0, request_start_ns=5_000 - ).to_data() + ) await processor.process_record(record) mask = processor.query_time_range(0, 10_000) assert mask.sum() == 1 @@ -529,9 +529,9 @@ async def test_single_record_outside( ) -> None: processor = MetricsAccumulator(mock_run) processor._tags_to_types = {} - record = create_metric_records_message( + record = create_metric_records_data( x_request_id="test-1", session_num=0, request_start_ns=15_000 - ).to_data() + ) await processor.process_record(record) mask = processor.query_time_range(0, 10_000) assert mask.sum() == 0 @@ -542,12 +542,12 @@ async def test_boundary_inclusive_start_exclusive_end( ) -> None: processor = MetricsAccumulator(mock_run) processor._tags_to_types = {} - record1 = create_metric_records_message( + record1 = create_metric_records_data( x_request_id="test-1", session_num=0, request_start_ns=1_000 - ).to_data() - record2 = create_metric_records_message( + ) + record2 = create_metric_records_data( x_request_id="test-2", session_num=1, request_start_ns=2_000 - ).to_data() + ) await processor.process_record(record1) await processor.process_record(record2) # [1_000, 2_000) should include 1_000 but exclude 2_000 @@ -563,9 +563,9 @@ async def test_multiple_records_filtering( processor = MetricsAccumulator(mock_run) processor._tags_to_types = {} for i, ts in enumerate([100, 200, 300, 400, 500]): - r = create_metric_records_message( + r = create_metric_records_data( x_request_id=f"test-{i}", session_num=i, request_start_ns=ts - ).to_data() + ) await processor.process_record(r) mask = processor.query_time_range(200, 400) @@ -584,12 +584,12 @@ async def test_summarize_returns_metrics_summary( processor._metric_classes = {RequestLatencyMetric.tag: RequestLatencyMetric} # Inject data via process_record - msg = create_metric_records_message( + msg = create_metric_records_data( x_request_id="test-1", session_num=0, results=[{RequestLatencyMetric.tag: 42.0}], ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) summary = await processor.summarize() @@ -669,23 +669,23 @@ async def test_summarize_with_timeslices( processor._metric_classes = {"test_record": RequestLatencyMetric} # Process records in two different 1-second windows - msg1 = create_metric_records_message( + msg1 = create_metric_records_data( x_request_id="test-1", session_num=0, request_start_ns=int(0.5 * NANOS_PER_SECOND), request_end_ns=int(0.6 * NANOS_PER_SECOND), results=[{"test_record": 42.0}], ) - await processor.process_record(msg1.to_data()) + await processor.process_record(msg1) - msg2 = create_metric_records_message( + msg2 = create_metric_records_data( x_request_id="test-2", session_num=1, request_start_ns=int(1.5 * NANOS_PER_SECOND), request_end_ns=int(2.5 * NANOS_PER_SECOND), results=[{"test_record": 84.0}], ) - await processor.process_record(msg2.to_data()) + await processor.process_record(msg2) summary = await processor.summarize() @@ -705,12 +705,12 @@ async def test_summarize_no_timeslices_without_config( processor._tags_to_types = {"test_record": MetricType.RECORD} processor._metric_classes = {"test_record": RequestLatencyMetric} - msg = create_metric_records_message( + msg = create_metric_records_data( x_request_id="test-1", session_num=0, results=[{"test_record": 42.0}], ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) summary = await processor.summarize() assert summary.timeslices is None @@ -726,21 +726,21 @@ async def test_timeslice_accumulation( processor._metric_classes = {"test_record": RequestLatencyMetric} # Two records in same 1-second window - msg1 = create_metric_records_message( + msg1 = create_metric_records_data( x_request_id="test-1", session_num=0, request_start_ns=int(0.3 * NANOS_PER_SECOND), results=[{"test_record": 10.0}], ) - await processor.process_record(msg1.to_data()) + await processor.process_record(msg1) - msg2 = create_metric_records_message( + msg2 = create_metric_records_data( x_request_id="test-2", session_num=1, request_start_ns=int(0.7 * NANOS_PER_SECOND), results=[{"test_record": 20.0}], ) - await processor.process_record(msg2.to_data()) + await processor.process_record(msg2) summary = await processor.summarize() assert summary.timeslices is not None @@ -761,33 +761,33 @@ async def test_timeslice_aggregate_metrics( processor._metric_classes = {RequestCountMetric.tag: RequestCountMetric} # First timeslice: 5 + 3 = 8 - msg1 = create_metric_records_message( + msg1 = create_metric_records_data( x_request_id="test-1", session_num=0, request_start_ns=int(0.5 * NANOS_PER_SECOND), request_end_ns=int(0.6 * NANOS_PER_SECOND), results=[{RequestCountMetric.tag: 5}], ) - await processor.process_record(msg1.to_data()) + await processor.process_record(msg1) - msg2 = create_metric_records_message( + msg2 = create_metric_records_data( x_request_id="test-2", session_num=1, request_start_ns=int(0.7 * NANOS_PER_SECOND), request_end_ns=int(0.8 * NANOS_PER_SECOND), results=[{RequestCountMetric.tag: 3}], ) - await processor.process_record(msg2.to_data()) + await processor.process_record(msg2) # Second timeslice: 7 - msg3 = create_metric_records_message( + msg3 = create_metric_records_data( x_request_id="test-3", session_num=2, request_start_ns=int(1.5 * NANOS_PER_SECOND), request_end_ns=int(2.5 * NANOS_PER_SECOND), results=[{RequestCountMetric.tag: 7}], ) - await processor.process_record(msg3.to_data()) + await processor.process_record(msg3) summary = await processor.summarize() assert summary.timeslices is not None @@ -810,21 +810,21 @@ async def test_timeslice_max_aggregate( processor._aggregation_kinds = {"max_ts": AggregationKind.MAX} processor._metric_classes = {"max_ts": RequestCountMetric} - msg1 = create_metric_records_message( + msg1 = create_metric_records_data( x_request_id="test-1", session_num=0, request_start_ns=int(0.3 * NANOS_PER_SECOND), results=[{"max_ts": 100}], ) - await processor.process_record(msg1.to_data()) + await processor.process_record(msg1) - msg2 = create_metric_records_message( + msg2 = create_metric_records_data( x_request_id="test-2", session_num=1, request_start_ns=int(0.7 * NANOS_PER_SECOND), results=[{"max_ts": 300}], ) - await processor.process_record(msg2.to_data()) + await processor.process_record(msg2) summary = await processor.summarize() assert summary.timeslices is not None @@ -842,21 +842,21 @@ async def test_timeslice_min_aggregate( processor._aggregation_kinds = {"min_ts": AggregationKind.MIN} processor._metric_classes = {"min_ts": RequestCountMetric} - msg1 = create_metric_records_message( + msg1 = create_metric_records_data( x_request_id="test-1", session_num=0, request_start_ns=int(0.3 * NANOS_PER_SECOND), results=[{"min_ts": 500}], ) - await processor.process_record(msg1.to_data()) + await processor.process_record(msg1) - msg2 = create_metric_records_message( + msg2 = create_metric_records_data( x_request_id="test-2", session_num=1, request_start_ns=int(0.7 * NANOS_PER_SECOND), results=[{"min_ts": 200}], ) - await processor.process_record(msg2.to_data()) + await processor.process_record(msg2) summary = await processor.summarize() assert summary.timeslices is not None @@ -873,23 +873,23 @@ async def test_compute_timeslices_populates_window_bounds( processor._tags_to_types = {"test_record": MetricType.RECORD} processor._metric_classes = {"test_record": RequestLatencyMetric} - msg1 = create_metric_records_message( + msg1 = create_metric_records_data( x_request_id="test-1", session_num=0, request_start_ns=int(0.5 * NANOS_PER_SECOND), request_end_ns=int(0.6 * NANOS_PER_SECOND), results=[{"test_record": 42.0}], ) - await processor.process_record(msg1.to_data()) + await processor.process_record(msg1) - msg2 = create_metric_records_message( + msg2 = create_metric_records_data( x_request_id="test-2", session_num=1, request_start_ns=int(1.5 * NANOS_PER_SECOND), request_end_ns=int(2.5 * NANOS_PER_SECOND), results=[{"test_record": 84.0}], ) - await processor.process_record(msg2.to_data()) + await processor.process_record(msg2) summary = await processor.summarize() @@ -914,12 +914,12 @@ async def test_timeslices_none_without_config( processor._tags_to_types = {"test_record": MetricType.RECORD} processor._metric_classes = {"test_record": RequestLatencyMetric} - msg = create_metric_records_message( + msg = create_metric_records_data( x_request_id="test-1", session_num=0, results=[{"test_record": 42.0}], ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) summary = await processor.summarize() assert summary.timeslices is None @@ -938,23 +938,23 @@ async def test_last_timeslice_clips_to_run_end( processor._metric_classes = {"test_record": RequestLatencyMetric} # Run extends from 0.5s to 1.7s — slice 1 [1.5, 2.5) overshoots. - msg1 = create_metric_records_message( + msg1 = create_metric_records_data( x_request_id="test-1", session_num=0, request_start_ns=int(0.5 * NANOS_PER_SECOND), request_end_ns=int(1.4 * NANOS_PER_SECOND), results=[{"test_record": 1.0}], ) - await processor.process_record(msg1.to_data()) + await processor.process_record(msg1) - msg2 = create_metric_records_message( + msg2 = create_metric_records_data( x_request_id="test-2", session_num=1, request_start_ns=int(1.5 * NANOS_PER_SECOND), request_end_ns=int(1.7 * NANOS_PER_SECOND), results=[{"test_record": 2.0}], ) - await processor.process_record(msg2.to_data()) + await processor.process_record(msg2) summary = await processor.summarize() assert summary.timeslices is not None @@ -1161,7 +1161,7 @@ async def test_timeslice_has_effective_concurrency_and_throughput( ) # One request: 0.5s start, 0.8s end, 10 output tokens, 50ms TTFT - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=0, request_start_ns=int(0.5 * NANOS_PER_SECOND), request_end_ns=int(0.8 * NANOS_PER_SECOND), @@ -1173,7 +1173,7 @@ async def test_timeslice_has_effective_concurrency_and_throughput( } ], ) - await acc.process_record(msg.to_data()) + await acc.process_record(msg) summary = await acc.summarize() assert summary.timeslices is not None @@ -1208,7 +1208,7 @@ async def test_timeslice_effective_concurrency_overlapping_requests( for i, (start, end) in enumerate( [(0.1, 1.5), (0.5, 1.8)], ): - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=i, request_start_ns=int(start * NANOS_PER_SECOND), request_end_ns=int(end * NANOS_PER_SECOND), @@ -1220,7 +1220,7 @@ async def test_timeslice_effective_concurrency_overlapping_requests( } ], ) - await acc.process_record(msg.to_data()) + await acc.process_record(msg) summary = await acc.summarize() assert summary.timeslices is not None @@ -1238,7 +1238,7 @@ async def test_timeslice_effective_throughput_nonzero( mock_run, latency_cls, output_cls, ttft_cls ) - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=0, request_start_ns=int(0.1 * NANOS_PER_SECOND), request_end_ns=int(0.9 * NANOS_PER_SECOND), @@ -1250,7 +1250,7 @@ async def test_timeslice_effective_throughput_nonzero( } ], ) - await acc.process_record(msg.to_data()) + await acc.process_record(msg) summary = await acc.summarize() assert summary.timeslices is not None @@ -1266,13 +1266,13 @@ async def test_timeslice_sweep_metrics_zero_throughput_without_tokens( latency_cls, _, _, _ = _make_sweep_metric_classes() acc = create_accumulator_with_metrics(mock_run, latency_cls) - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=0, request_start_ns=int(0.2 * NANOS_PER_SECOND), request_end_ns=int(0.7 * NANOS_PER_SECOND), results=[{"request_latency": 500_000_000.0}], ) - await acc.process_record(msg.to_data()) + await acc.process_record(msg) summary = await acc.summarize() assert summary.timeslices is not None @@ -1298,7 +1298,7 @@ async def test_timeslice_sweep_metrics_multiple_slices( (2, 2.1, 2.9, 800e6, 30.0, 50e6), ] for session_num, start, end, latency, tokens, ttft in records: - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=session_num, request_start_ns=int(start * NANOS_PER_SECOND), request_end_ns=int(end * NANOS_PER_SECOND), @@ -1310,7 +1310,7 @@ async def test_timeslice_sweep_metrics_multiple_slices( } ], ) - await acc.process_record(msg.to_data()) + await acc.process_record(msg) summary = await acc.summarize() assert summary.timeslices is not None @@ -1346,7 +1346,7 @@ async def test_overall_has_effective_concurrency_and_throughput( mock_run, latency_cls, output_cls, ttft_cls ) - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=0, request_start_ns=int(0.1 * NANOS_PER_SECOND), request_end_ns=int(0.9 * NANOS_PER_SECOND), @@ -1358,7 +1358,7 @@ async def test_overall_has_effective_concurrency_and_throughput( } ], ) - await acc.process_record(msg.to_data()) + await acc.process_record(msg) summary = await acc.summarize() assert "effective_concurrency" in summary.results @@ -1385,7 +1385,7 @@ async def test_overall_effective_concurrency_overlapping_requests( ) for i, (start, end) in enumerate([(0.1, 1.5), (0.5, 1.8)]): - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=i, request_start_ns=int(start * NANOS_PER_SECOND), request_end_ns=int(end * NANOS_PER_SECOND), @@ -1397,7 +1397,7 @@ async def test_overall_effective_concurrency_overlapping_requests( } ], ) - await acc.process_record(msg.to_data()) + await acc.process_record(msg) summary = await acc.summarize() assert summary.results["effective_concurrency"].avg > 1.0 @@ -1412,7 +1412,7 @@ async def test_overall_effective_throughput_nonzero( mock_run, latency_cls, output_cls, ttft_cls ) - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=0, request_start_ns=int(0.1 * NANOS_PER_SECOND), request_end_ns=int(0.9 * NANOS_PER_SECOND), @@ -1424,7 +1424,7 @@ async def test_overall_effective_throughput_nonzero( } ], ) - await acc.process_record(msg.to_data()) + await acc.process_record(msg) summary = await acc.summarize() assert summary.results["effective_decode_throughput"].avg > 0.0 @@ -1437,13 +1437,13 @@ async def test_overall_zero_throughput_without_tokens( latency_cls, _, _, _ = _make_sweep_metric_classes() acc = create_accumulator_with_metrics(mock_run, latency_cls) - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=0, request_start_ns=int(0.2 * NANOS_PER_SECOND), request_end_ns=int(0.7 * NANOS_PER_SECOND), results=[{"request_latency": 500_000_000.0}], ) - await acc.process_record(msg.to_data()) + await acc.process_record(msg) summary = await acc.summarize() assert summary.results["effective_decode_throughput"].avg == 0.0 @@ -1472,7 +1472,7 @@ async def test_overall_effective_prefill_throughput_nonzero( mock_run, latency_cls, output_cls, ttft_cls, isl_cls ) - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=0, request_start_ns=int(0.1 * NANOS_PER_SECOND), request_end_ns=int(0.9 * NANOS_PER_SECOND), @@ -1485,7 +1485,7 @@ async def test_overall_effective_prefill_throughput_nonzero( } ], ) - await acc.process_record(msg.to_data()) + await acc.process_record(msg) summary = await acc.summarize() assert summary.results["effective_prefill_throughput"].avg > 0.0 @@ -1500,7 +1500,7 @@ async def test_overall_zero_prefill_throughput_without_isl( mock_run, latency_cls, output_cls, ttft_cls ) - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=0, request_start_ns=int(0.2 * NANOS_PER_SECOND), request_end_ns=int(0.7 * NANOS_PER_SECOND), @@ -1512,7 +1512,7 @@ async def test_overall_zero_prefill_throughput_without_isl( } ], ) - await acc.process_record(msg.to_data()) + await acc.process_record(msg) summary = await acc.summarize() assert summary.results["effective_prefill_throughput"].avg == 0.0 @@ -1533,10 +1533,10 @@ async def test_default_backend_is_ragged( processor = MetricsAccumulator(mock_run) processor._tags_to_types = {"test_list": MetricType.RECORD} - message = create_metric_records_message( + message = create_metric_records_data( session_num=0, results=[{"test_list": [1.0, 2.0, 3.0]}] ) - await processor.process_record(message.to_data()) + await processor.process_record(message) backend = processor._column_store.ragged("test_list") assert isinstance(backend, RaggedSeries) @@ -1556,10 +1556,10 @@ async def test_tdigest_backend_via_env_flag( processor = MetricsAccumulator(mock_run) processor._tags_to_types = {"test_list": MetricType.RECORD} - message = create_metric_records_message( + message = create_metric_records_data( session_num=0, results=[{"test_list": [10.0, 20.0, 30.0, 40.0]}] ) - await processor.process_record(message.to_data()) + await processor.process_record(message) backend = processor._column_store.ragged("test_list") assert isinstance(backend, TDigestListMetricAggregator) @@ -1595,10 +1595,10 @@ async def _run(backend_name: str) -> MetricResult: ) } for i, lst in enumerate(chunk_lists): - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=i, results=[{"inter_chunk_latency": lst}] ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) results = processor._compute_results() return results["inter_chunk_latency"] @@ -1639,11 +1639,11 @@ async def test_tdigest_skips_per_record_replay_in_sweeps( processor = MetricsAccumulator(mock_run) processor._tags_to_types = {"inter_chunk_latency": MetricType.RECORD} - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=0, results=[{"inter_chunk_latency": [10.0, 20.0, 30.0]}], ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) # ICL is recorded but the backend doesn't support replay. assert "inter_chunk_latency" in processor._column_store.ragged_tags() @@ -1675,8 +1675,8 @@ async def test_bool_metadata_round_trip( benchmark_phase="profiling", was_cancelled=cancelled, ) - msg = create_metric_records_message(metadata=meta) - await processor.process_record(msg.to_data()) + msg = create_metric_records_data(metadata=meta) + await processor.process_record(msg) store = processor._column_store # has_error and was_cancelled should now be in _metadata_bool, not _metadata_numeric @@ -1702,12 +1702,12 @@ async def test_categorical_metadata_intern_pool( worker_ids = ["worker_a", "worker_b", "worker_a", "worker_c", "worker_b"] for i, wid in enumerate(worker_ids): - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=i, request_start_ns=1_000_000_000 + i, worker_id=wid, ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) store = processor._column_store assert "worker_id" in store._metadata_categorical @@ -1736,13 +1736,13 @@ async def test_uuid_routing_drops_request_id_categoricalises_correlation_and_con ``mask_for_categorical``.""" processor = MetricsAccumulator(mock_run) - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=0, x_request_id="req-deadbeef", x_correlation_id="corr-cafebabe", conversation_id="conv-12345", ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) store = processor._column_store # x_request_id no longer stored anywhere — exporters read it off @@ -1766,12 +1766,12 @@ async def test_categorical_grouping_accessors( # Three "conversations" interleaved across six records correlations = ["conv_a", "conv_b", "conv_a", "conv_c", "conv_b", "conv_a"] for i, cid in enumerate(correlations): - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=i, request_start_ns=1_000_000_000 + i, x_correlation_id=cid, ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) store = processor._column_store # Enumerate unique values @@ -1819,8 +1819,8 @@ async def test_credit_to_start_and_effective_latency_present( benchmark_phase="profiling", turn_index=0, ) - msg = create_metric_records_message(metadata=meta) - await processor.process_record(msg.to_data()) + msg = create_metric_records_data(metadata=meta) + await processor.process_record(msg) summary = await processor.summarize() assert "credit_to_start_latency" in summary.results @@ -2017,14 +2017,14 @@ async def _ingest( phase: CreditPhase = CreditPhase.PROFILING, ) -> None: for i, start_ns in enumerate(starts): - msg = create_metric_records_message( + msg = create_metric_records_data( session_num=i, benchmark_phase=phase, request_start_ns=start_ns, request_end_ns=start_ns + 1_000_000, results=[{RequestCountMetric.tag: 1}], ) - await processor.process_record(msg.to_data()) + await processor.process_record(msg) @pytest.mark.asyncio async def test_export_results_straggler_stamped_at_window_end_included( diff --git a/tests/unit/post_processors/test_network_adjusted_metrics.py b/tests/unit/post_processors/test_network_adjusted_metrics.py index 51218739a0..856fdc5469 100644 --- a/tests/unit/post_processors/test_network_adjusted_metrics.py +++ b/tests/unit/post_processors/test_network_adjusted_metrics.py @@ -20,7 +20,7 @@ TimeToFirstOutputTokenMetric, ) from aiperf.metrics.types.ttft_metric import TTFTMetric -from tests.unit.post_processors.conftest import create_metric_records_message +from tests.unit.post_processors.conftest import create_metric_records_data _PERCENTILE_ATTRS = ("p1", "p5", "p10", "p25", "p50", "p75", "p90", "p95", "p99") _SHIFT_ATTRS = ("min", "max", "avg", *_PERCENTILE_ATTRS) @@ -49,12 +49,12 @@ async def _seed_records( for idx in range(length): metrics = {tag: values[idx] for tag, values in series_by_tag.items()} await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id=f"r-{idx}", request_start_ns=1_000_000_000 + idx, request_end_ns=1_100_000_000 + idx, results=[metrics], - ).to_data() + ) ) @@ -335,12 +335,12 @@ async def _seed_two_windows(self, accumulator: MetricsAccumulator) -> None: latencies = [4_000_000.0, 6_000_000.0, 4_000_000.0, 6_000_000.0] for idx, (start, lat) in enumerate(zip(starts, latencies, strict=True)): await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id=f"r-{idx}", request_start_ns=start, request_end_ns=start + int(lat), results=[{RequestLatencyMetric.tag: lat, TTFTMetric.tag: lat / 2}], - ).to_data() + ) ) async def test_network_adjusted_present_in_every_timeslice(self, mock_run) -> None: diff --git a/tests/unit/post_processors/test_otel_metrics_results_processor.py b/tests/unit/post_processors/test_otel_metrics_results_processor.py index 0e1da840f4..6e4bf07e6c 100644 --- a/tests/unit/post_processors/test_otel_metrics_results_processor.py +++ b/tests/unit/post_processors/test_otel_metrics_results_processor.py @@ -21,7 +21,7 @@ OTelMetricsResultsProcessor, ) from tests.unit.conftest import make_run_from_cli -from tests.unit.post_processors.conftest import create_metric_records_message +from tests.unit.post_processors.conftest import create_metric_records_data @pytest.fixture @@ -188,7 +188,7 @@ async def test_process_record_records_histogram_values_by_metric( ) fake_queue = _setup_fanout_processor(processor) - metric_record = create_metric_records_message( + metric_record = create_metric_records_data( results=[ { "request_latency_ns": 123_000_000, @@ -196,7 +196,7 @@ async def test_process_record_records_histogram_values_by_metric( "tokens_per_response": [1, 2, 3], } ] - ).to_data() + ) await processor.process_record(metric_record) histogram_events = [ @@ -233,9 +233,9 @@ async def test_process_record_skips_metrics_when_metrics_telemetry_disabled( ) fake_queue = _setup_fanout_processor(processor) - metric_record = create_metric_records_message( + metric_record = create_metric_records_data( results=[{"request_latency_ns": 123_000_000}] - ).to_data() + ) await processor.process_record(metric_record) histogram_events = [ @@ -557,9 +557,9 @@ async def test_process_record_fanout_emits_metric_and_timing_events( ) fake_queue = _setup_fanout_processor(processor) - metric_record = create_metric_records_message( + metric_record = create_metric_records_data( results=[{"request_latency_ns": 123_000_000, "request_count": 1}] - ).to_data() + ) await processor.process_record(metric_record) timing_stats = CreditPhaseStats( @@ -697,9 +697,9 @@ def test_build_record_attributes( service_id="records-manager", run=cfg_otel, ) - metric_record = create_metric_records_message( + metric_record = create_metric_records_data( results=[{"request_latency_ns": 123_000_000}] - ).to_data() + ) attributes = processor.build_record_attributes(metric_record) assert attributes["aiperf.worker.id"] == metric_record.metadata.worker_id diff --git a/tests/unit/post_processors/test_outputs_json_record_processor.py b/tests/unit/post_processors/test_outputs_json_record_processor.py index 248185bef4..3341909c06 100644 --- a/tests/unit/post_processors/test_outputs_json_record_processor.py +++ b/tests/unit/post_processors/test_outputs_json_record_processor.py @@ -17,6 +17,7 @@ from aiperf.post_processors.outputs_json_record_processor import ( OutputsJsonRecordProcessor, ) +from aiperf.post_processors.record_observer_context import RecordObserverContext from tests.unit.post_processors.conftest import aiperf_lifecycle @@ -93,7 +94,9 @@ async def test_process_record_writes_fragment(self, tmp_path: Path) -> None: run=MagicMock(cfg=config), ) async with aiperf_lifecycle(processor) as proc: - await proc.process_record(record, metadata) + await proc.observe( + RecordObserverContext(record=record, metadata=metadata, produced={}) + ) assert proc.lines_written == 1 @@ -125,7 +128,9 @@ async def test_process_record_extracts_response_text(self, tmp_path: Path) -> No run=MagicMock(cfg=config), ) async with aiperf_lifecycle(processor) as proc: - await proc.process_record(record, metadata) + await proc.observe( + RecordObserverContext(record=record, metadata=metadata, produced={}) + ) # Read the written fragment file and verify response_text output_file = proc.output_file @@ -159,7 +164,9 @@ async def test_process_record_null_response_text_when_no_content( run=MagicMock(cfg=config), ) async with aiperf_lifecycle(processor) as proc: - await proc.process_record(record, metadata) + await proc.observe( + RecordObserverContext(record=record, metadata=metadata, produced={}) + ) # Read the written fragment file and verify response_text is absent (exclude_none=True) output_file = proc.output_file diff --git a/tests/unit/post_processors/test_post_processor_integration.py b/tests/unit/post_processors/test_post_processor_integration.py index 3102944d48..ce17293b7f 100644 --- a/tests/unit/post_processors/test_post_processor_integration.py +++ b/tests/unit/post_processors/test_post_processor_integration.py @@ -18,7 +18,7 @@ from aiperf.metrics.types.request_throughput_metric import RequestThroughputMetric from aiperf.post_processors.metric_record_processor import MetricRecordProcessor from tests.unit.post_processors.conftest import ( - create_metric_records_message, + create_metric_records_data, setup_mock_registry_sequences, ) @@ -35,14 +35,14 @@ class TestPostProcessorIntegration: async def test_record_to_accumulator_data_flow(self, mock_run) -> None: """MetricRecordsData flows into the accumulator summary.""" accumulator = MetricsAccumulator(mock_run) - message = create_metric_records_message( + message = create_metric_records_data( x_request_id="test-1", results=[ {RequestLatencyMetric.tag: 100_000_000.0, RequestCountMetric.tag: 1} ], ) - await accumulator.process_record(message.to_data()) + await accumulator.process_record(message) summary = await accumulator.summarize() assert summary.results[RequestLatencyMetric.tag].avg == pytest.approx(100.0) @@ -54,13 +54,13 @@ async def test_multiple_batches_accumulation(self, mock_run) -> None: accumulator = MetricsAccumulator(mock_run) for idx, value in enumerate(TEST_LATENCY_VALUES_NS): - message = create_metric_records_message( + message = create_metric_records_data( x_request_id=f"test-{idx}", request_start_ns=1_000_000_000 + idx, x_correlation_id=f"test-correlation-{idx}", results=[{RequestLatencyMetric.tag: value}], ) - await accumulator.process_record(message.to_data()) + await accumulator.process_record(message) summary = await accumulator.summarize() latency = summary.results[RequestLatencyMetric.tag] @@ -90,14 +90,14 @@ async def test_error_metrics_isolation( metadata = create_metric_metadata() result = await record_processor.process_record(error_parsed_record, metadata) - assert ErrorRequestCountMetric.tag in result - assert result[ErrorRequestCountMetric.tag] == 1 + assert ErrorRequestCountMetric.tag in result.metrics + assert result.metrics[ErrorRequestCountMetric.tag] == 1 async def test_derived_metrics_computation(self, mock_run) -> None: """Derived metrics are computed from accumulated aggregate metrics.""" accumulator = MetricsAccumulator(mock_run) await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id="test-1", results=[ { @@ -108,7 +108,7 @@ async def test_derived_metrics_computation(self, mock_run) -> None: ), } ], - ).to_data() + ) ) summary = await accumulator.summarize() @@ -124,11 +124,11 @@ async def test_complete_pipeline_summary(self, mock_run) -> None: for idx, value in enumerate(TEST_LATENCY_VALUES_NS): await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id=f"test-{idx}", request_start_ns=1_000_000_000 + idx, results=[{RequestLatencyMetric.tag: value}], - ).to_data() + ) ) summary = await accumulator.summarize() diff --git a/tests/unit/post_processors/test_raw_record_writer_adversarial.py b/tests/unit/post_processors/test_raw_record_writer_adversarial.py index 42b01b8266..6aa45f48c1 100644 --- a/tests/unit/post_processors/test_raw_record_writer_adversarial.py +++ b/tests/unit/post_processors/test_raw_record_writer_adversarial.py @@ -37,6 +37,7 @@ RawRecordAggregator, RawRecordWriterProcessor, ) +from aiperf.post_processors.record_observer_context import RecordObserverContext from tests.unit.post_processors.conftest import ( create_exporter_config, create_metric_metadata, @@ -439,19 +440,28 @@ async def test_aggregator_unlinks_inputs_after_concat_always( # Build three processor files with one record each async with raw_record_processor("processor-A", run_raw) as proc_a: - await proc_a.process_record( - sample_parsed_record_with_raw_responses, - create_metric_metadata(conversation_id="c-a"), + await proc_a.observe( + RecordObserverContext( + record=sample_parsed_record_with_raw_responses, + metadata=create_metric_metadata(conversation_id="c-a"), + produced={}, + ) ) async with raw_record_processor("processor-B", run_raw) as proc_b: - await proc_b.process_record( - sample_parsed_record_with_raw_responses, - create_metric_metadata(conversation_id="c-b"), + await proc_b.observe( + RecordObserverContext( + record=sample_parsed_record_with_raw_responses, + metadata=create_metric_metadata(conversation_id="c-b"), + produced={}, + ) ) async with raw_record_processor("processor-C", run_raw) as proc_c: - await proc_c.process_record( - sample_parsed_record_with_raw_responses, - create_metric_metadata(conversation_id="c-c"), + await proc_c.observe( + RecordObserverContext( + record=sample_parsed_record_with_raw_responses, + metadata=create_metric_metadata(conversation_id="c-c"), + produced={}, + ) ) inputs_before = sorted(raw_dir.glob("raw_records_*.jsonl")) diff --git a/tests/unit/post_processors/test_raw_record_writer_processor.py b/tests/unit/post_processors/test_raw_record_writer_processor.py index 2707455021..4cda4f7acb 100644 --- a/tests/unit/post_processors/test_raw_record_writer_processor.py +++ b/tests/unit/post_processors/test_raw_record_writer_processor.py @@ -13,6 +13,7 @@ RawRecordAggregator, RawRecordWriterProcessor, ) +from aiperf.post_processors.record_observer_context import RecordObserverContext from tests.unit.post_processors.conftest import ( create_exporter_config, create_metric_metadata, @@ -96,7 +97,11 @@ async def test_process_record_writes_valid_data( x_correlation_id="corr-123", ) - await processor.process_record(sample_parsed_record, metadata) + await processor.observe( + RecordObserverContext( + record=sample_parsed_record, metadata=metadata, produced={} + ) + ) assert processor.output_file.exists() lines = processor.output_file.read_text().splitlines() @@ -128,7 +133,11 @@ async def test_process_record_with_error( conversation_id="conv-error", ) - await processor.process_record(error_parsed_record, metadata) + await processor.observe( + RecordObserverContext( + record=error_parsed_record, metadata=metadata, produced={} + ) + ) record_dict = orjson.loads(processor.output_file.read_text().splitlines()[0]) @@ -155,7 +164,11 @@ async def test_process_multiple_records( conversation_id=f"conv-{i}", x_request_id=f"req-{i}", ) - await processor.process_record(sample_parsed_record, metadata) + await processor.observe( + RecordObserverContext( + record=sample_parsed_record, metadata=metadata, produced={} + ) + ) assert processor.lines_written == 5 lines = processor.output_file.read_text().splitlines() @@ -185,7 +198,11 @@ async def test_output_is_valid_jsonl_and_record_structure( turn_index=2, ) - await processor.process_record(sample_parsed_record, metadata) + await processor.observe( + RecordObserverContext( + record=sample_parsed_record, metadata=metadata, produced={} + ) + ) lines = processor.output_file.read_text().splitlines() @@ -227,7 +244,11 @@ async def test_aggregator_combines_multiple_files( session_num=i * 2 + j, conversation_id=f"conv-{i}-{j}", ) - await processor.process_record(sample_parsed_record, metadata) + await processor.observe( + RecordObserverContext( + record=sample_parsed_record, metadata=metadata, produced={} + ) + ) # Run aggregator exporter_config = create_exporter_config(cfg_raw) @@ -303,7 +324,11 @@ async def test_aggregator_clears_existing_output( # Create a processor file async with raw_record_processor("processor-1", run_raw) as processor: metadata = create_metric_metadata() - await processor.process_record(sample_parsed_record, metadata) + await processor.observe( + RecordObserverContext( + record=sample_parsed_record, metadata=metadata, produced={} + ) + ) # Run aggregator exporter_config = create_exporter_config(cfg_raw) diff --git a/tests/unit/post_processors/test_record_export_jsonl_writer_detailed.py b/tests/unit/post_processors/test_record_export_jsonl_writer_detailed.py index 0eeebdf24c..47dfd4306a 100644 --- a/tests/unit/post_processors/test_record_export_jsonl_writer_detailed.py +++ b/tests/unit/post_processors/test_record_export_jsonl_writer_detailed.py @@ -11,7 +11,7 @@ from aiperf.common.enums import CreditPhase, ExportLevel from aiperf.common.environment import Environment from aiperf.common.exceptions import PostProcessorDisabled -from aiperf.common.messages import MetricRecordsMessage +from aiperf.common.messages import MetricRecordsData from aiperf.common.models.record_models import ( MetricRecordInfo, MetricRecordMetadata, @@ -26,7 +26,7 @@ ) from tests.unit.post_processors.conftest import ( aiperf_lifecycle, - create_metric_records_message, + create_metric_records_data, ) @@ -78,8 +78,8 @@ def cli_config() -> CLIConfig: @pytest.fixture def sample_metric_records_message(): - """Create a sample MetricRecordsMessage for testing.""" - return create_metric_records_message( + """Create a sample MetricRecordsData for testing.""" + return create_metric_records_data( service_id="processor-1", x_request_id="test-record-123", conversation_id="conv-456", @@ -201,7 +201,7 @@ class TestRecordExportJSONLWriterProcessResult: async def test_process_record_writes_valid_data( self, run_records_export, - sample_metric_records_message: MetricRecordsMessage, + sample_metric_records_message: MetricRecordsData, mock_metric_registry: Mock, ): """Test that process_record writes valid data to file.""" @@ -221,7 +221,7 @@ async def test_process_record_writes_valid_data( "to_display_dict", return_value=mock_display_dict, ): - await processor.process_record(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message) lines = processor.output_file.read_text().splitlines() @@ -243,7 +243,7 @@ async def test_process_record_writes_valid_data( async def test_process_record_with_empty_display_metrics( self, run_records_export, - sample_metric_records_message: MetricRecordsMessage, + sample_metric_records_message: MetricRecordsData, mock_metric_registry: Mock, ): """Test that process_record skips records with empty display metrics.""" @@ -254,7 +254,7 @@ async def test_process_record_with_empty_display_metrics( # Mock to_display_dict to return empty dict with patch.object(MetricRecordDict, "to_display_dict", return_value={}): - await processor.process_record(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message) # Should not write anything since display_metrics is empty assert processor.lines_written == 0 @@ -266,7 +266,7 @@ async def test_process_record_with_empty_display_metrics( async def test_process_record_handles_errors_gracefully( self, run_records_export, - sample_metric_records_message: MetricRecordsMessage, + sample_metric_records_message: MetricRecordsData, mock_metric_registry: Mock, ): """Test that errors during processing don't raise exceptions.""" @@ -283,7 +283,7 @@ async def test_process_record_handles_errors_gracefully( patch.object(processor, "error") as mock_error, ): # Should not raise - await processor.process_record(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message) # Should log the error assert mock_error.call_count >= 1 @@ -295,7 +295,7 @@ async def test_process_record_handles_errors_gracefully( async def test_process_record_multiple_messages( self, run_records_export, - sample_metric_records_message: MetricRecordsMessage, + sample_metric_records_message: MetricRecordsData, mock_metric_registry: Mock, ): """Test processing multiple messages accumulates records.""" @@ -313,14 +313,14 @@ async def test_process_record_multiple_messages( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): for i in range(5): - message = create_metric_records_message( + message = create_metric_records_data( x_request_id=f"record-{i}", conversation_id=f"conv-{i}", turn_index=i, request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}, {"metric2": 200}], ) - await processor.process_record(message.to_data()) + await processor.process_record(message) assert processor.lines_written == 5 assert processor.output_file.exists() @@ -344,7 +344,7 @@ class TestRecordExportJSONLWriterFileFormat: async def test_output_is_valid_jsonl( self, run_records_export, - sample_metric_records_message: MetricRecordsMessage, + sample_metric_records_message: MetricRecordsData, mock_metric_registry: Mock, ): """Test that output file is valid JSONL format.""" @@ -359,7 +359,7 @@ async def test_output_is_valid_jsonl( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_record(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message) lines = processor.output_file.read_text().splitlines() @@ -374,7 +374,7 @@ async def test_output_is_valid_jsonl( async def test_record_structure_is_complete( self, run_records_export, - sample_metric_records_message: MetricRecordsMessage, + sample_metric_records_message: MetricRecordsData, mock_metric_registry: Mock, ): """Test that each record has the expected structure.""" @@ -389,7 +389,7 @@ async def test_record_structure_is_complete( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_record(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message) lines = processor.output_file.read_text().splitlines() @@ -437,14 +437,14 @@ async def test_periodic_debug_logging( ): with caplog.at_level(logging.DEBUG): for i in range(processor._batch_size): - message = create_metric_records_message( + message = create_metric_records_data( x_request_id=f"record-{i}", conversation_id=f"conv-{i}", turn_index=i, request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}, {"metric2": 200}], ) - await processor.process_record(message.to_data()) + await processor.process_record(message) # Wait for async flush task to complete await processor.wait_for_tasks() @@ -456,7 +456,7 @@ async def test_periodic_debug_logging( async def test_error_logging_on_write_failure( self, run_records_export, - sample_metric_records_message: MetricRecordsMessage, + sample_metric_records_message: MetricRecordsData, mock_metric_registry: Mock, ): """Test that errors are logged when write fails.""" @@ -471,7 +471,7 @@ async def test_error_logging_on_write_failure( ), patch.object(processor, "error") as mock_error, ): - await processor.process_record(sample_metric_records_message.to_data()) + await processor.process_record(sample_metric_records_message) assert mock_error.call_count >= 1 call_args = str(mock_error.call_args_list[0]) @@ -485,7 +485,7 @@ class TestRecordExportJSONLWriterShutdown: async def test_shutdown_logs_statistics( self, run_records_export, - sample_metric_records_message: MetricRecordsMessage, + sample_metric_records_message: MetricRecordsData, mock_metric_registry: Mock, ): """Test that shutdown logs final statistics.""" @@ -504,14 +504,14 @@ async def test_shutdown_logs_statistics( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): for i in range(3): - message = create_metric_records_message( + message = create_metric_records_data( x_request_id=f"record-{i}", conversation_id=f"conv-{i}", turn_index=i, request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}], ) - await processor.process_record(message.to_data()) + await processor.process_record(message) # Wait for any pending flush tasks await processor.wait_for_tasks() @@ -666,7 +666,7 @@ async def test_trace_data_excluded_when_disabled( ) # Create message with trace_data - message = create_metric_records_message( + message = create_metric_records_data( x_request_id="test-record-with-trace", conversation_id="conv-trace-1", results=[{"test_metric": 42}], @@ -677,7 +677,7 @@ async def test_trace_data_excluded_when_disabled( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_record(message.to_data()) + await processor.process_record(message) lines = processor.output_file.read_text().splitlines() assert len(lines) == 1 @@ -706,7 +706,7 @@ async def test_trace_data_included_when_enabled( ) # Create message with trace_data - message = create_metric_records_message( + message = create_metric_records_data( x_request_id="test-record-with-trace", conversation_id="conv-trace-2", results=[{"test_metric": 42}], @@ -717,7 +717,7 @@ async def test_trace_data_included_when_enabled( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_record(message.to_data()) + await processor.process_record(message) lines = processor.output_file.read_text().splitlines() assert len(lines) == 1 @@ -760,7 +760,7 @@ async def test_metrics_always_present_regardless_of_trace_flag( ) for processor in [processor_disabled, processor_enabled]: - message = create_metric_records_message( + message = create_metric_records_data( x_request_id="test-record-metrics", conversation_id="conv-metrics", results=[{"request_latency_ns": 100_500_000, "output_token_count": 50}], @@ -771,7 +771,7 @@ async def test_metrics_always_present_regardless_of_trace_flag( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_record(message.to_data()) + await processor.process_record(message) lines = processor.output_file.read_text().splitlines() assert len(lines) == 1 @@ -800,7 +800,7 @@ async def test_no_trace_data_when_record_has_none( ) # Create message WITHOUT trace_data - message = create_metric_records_message( + message = create_metric_records_data( x_request_id="test-record-no-trace", conversation_id="conv-no-trace", results=[{"test_metric": 42}], @@ -811,7 +811,7 @@ async def test_no_trace_data_when_record_has_none( with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_record(message.to_data()) + await processor.process_record(message) lines = processor.output_file.read_text().splitlines() assert len(lines) == 1 @@ -852,13 +852,13 @@ async def test_lifecycle( ): for i in range(Environment.RECORD.EXPORT_BATCH_SIZE * 2): await processor.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id=f"record-{i}", conversation_id=f"conv-{i}", turn_index=0, request_start_ns=1_000_000_000 + i, results=[{"inter_token_latency": 100}], - ).to_data() + ) ) # Wait for all async flush tasks to complete diff --git a/tests/unit/post_processors/test_timeslice_metric_results_processor.py b/tests/unit/post_processors/test_timeslice_metric_results_processor.py index 05a187ff49..f1f121a794 100644 --- a/tests/unit/post_processors/test_timeslice_metric_results_processor.py +++ b/tests/unit/post_processors/test_timeslice_metric_results_processor.py @@ -21,7 +21,7 @@ from aiperf.metrics.types.request_count_metric import RequestCountMetric from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric from aiperf.metrics.types.request_throughput_metric import RequestThroughputMetric -from tests.unit.post_processors.conftest import create_metric_records_message +from tests.unit.post_processors.conftest import create_metric_records_data @pytest.fixture @@ -76,20 +76,20 @@ async def test_process_record_separates_by_timeslice(self, mock_run) -> None: accumulator = MetricsAccumulator(mock_run) await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id="test-1", request_start_ns=int(0.5 * NANOS_PER_SECOND), request_end_ns=int(0.6 * NANOS_PER_SECOND), results=[{RequestLatencyMetric.tag: 42_000_000.0}], - ).to_data() + ) ) await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id="test-2", request_start_ns=int(1.5 * NANOS_PER_SECOND), request_end_ns=int(1.6 * NANOS_PER_SECOND), results=[{RequestLatencyMetric.tag: 84_000_000.0}], - ).to_data() + ) ) timeslices = (await accumulator.summarize()).timeslices @@ -106,12 +106,12 @@ async def test_process_record_accumulates_in_same_timeslice(self, mock_run) -> N for idx, value in enumerate([10_000_000.0, 20_000_000.0]): await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id=f"test-{idx}", request_start_ns=int((0.3 + idx * 0.4) * NANOS_PER_SECOND), request_end_ns=int((0.35 + idx * 0.4) * NANOS_PER_SECOND), results=[{RequestLatencyMetric.tag: value}], - ).to_data() + ) ) timeslices = (await accumulator.summarize()).timeslices @@ -134,12 +134,12 @@ async def test_aggregate_metric_per_timeslice(self, mock_run) -> None: ] for idx, (start_s, count) in enumerate(records): await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id=f"test-{idx}", request_start_ns=int(start_s * NANOS_PER_SECOND), request_end_ns=int((start_s + 0.1) * NANOS_PER_SECOND), results=[{RequestCountMetric.tag: count}], - ).to_data() + ) ) timeslices = (await accumulator.summarize()).timeslices @@ -160,12 +160,12 @@ async def test_timeslice_boundary_conditions(self, mock_run) -> None: ] for idx, (start_s, value) in enumerate(records): await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id=f"test-{idx}", request_start_ns=int(start_s * NANOS_PER_SECOND), request_end_ns=int((start_s + 0.01) * NANOS_PER_SECOND), results=[{RequestLatencyMetric.tag: value}], - ).to_data() + ) ) timeslices = (await accumulator.summarize()).timeslices @@ -187,7 +187,7 @@ async def test_derived_metrics_are_computed_per_timeslice(self, mock_run) -> Non ] for idx, (start_s, end_s, count) in enumerate(records): await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id=f"test-{idx}", request_start_ns=int(start_s * NANOS_PER_SECOND), request_end_ns=int(end_s * NANOS_PER_SECOND), @@ -198,7 +198,7 @@ async def test_derived_metrics_are_computed_per_timeslice(self, mock_run) -> Non MaxResponseTimestampMetric.tag: end_s * NANOS_PER_SECOND, } ], - ).to_data() + ) ) timeslices = (await accumulator.summarize()).timeslices @@ -228,12 +228,12 @@ async def test_multiple_timeslices_with_different_slice_duration( for i in range(4): start_s = i * 0.5 + 0.25 await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id=f"test-{i}", request_start_ns=int(start_s * NANOS_PER_SECOND), request_end_ns=int((start_s + 0.05) * NANOS_PER_SECOND), results=[{RequestLatencyMetric.tag: float(i * 1_000_000)}], - ).to_data() + ) ) timeslices = (await accumulator.summarize()).timeslices @@ -250,12 +250,12 @@ async def test_trailing_partial_timeslice_is_flagged(self, mock_run) -> None: accumulator = MetricsAccumulator(mock_run) await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id="test-1", request_start_ns=int(0.5 * NANOS_PER_SECOND), request_end_ns=int(0.6 * NANOS_PER_SECOND), results=[{RequestLatencyMetric.tag: 42_000_000.0}], - ).to_data() + ) ) timeslices = (await accumulator.summarize()).timeslices @@ -296,14 +296,14 @@ async def test_run_scoped_tags_never_derived_in_timeslice_results( # Two records straddling the 1s slice boundary so timeslices are built. for i, start_s in enumerate((0.2, 1.2)): await accumulator.process_record( - create_metric_records_message( + create_metric_records_data( x_request_id=f"req-{i}", request_start_ns=int(start_s * NANOS_PER_SECOND), request_end_ns=int((start_s + 0.05) * NANOS_PER_SECOND), results=[ {ReplaySendScheduleOffsetMetric.tag: i * NANOS_PER_MILLIS} ], - ).to_data() + ) ) summary = await accumulator.summarize() diff --git a/tests/unit/post_processors/test_timing_results_strategy.py b/tests/unit/post_processors/test_timing_results_strategy.py index 7c95a005f9..0477651e30 100644 --- a/tests/unit/post_processors/test_timing_results_strategy.py +++ b/tests/unit/post_processors/test_timing_results_strategy.py @@ -11,7 +11,7 @@ from aiperf.common.enums import CreditPhase from aiperf.common.models import CreditPhaseStats from aiperf.post_processors.strategies.timing_results import TimingResultsStrategy -from tests.unit.post_processors.conftest import create_metric_records_message +from tests.unit.post_processors.conftest import create_metric_records_data @dataclass @@ -118,9 +118,7 @@ class TestTimingResultsStrategy: def test_supports_only_credit_phase_stats(self) -> None: strategy = TimingResultsStrategy(_TimingStrategyContext(attributes={})) timing_stats = _create_credit_phase_stats() - metric_record = create_metric_records_message( - results=[{"request_latency_ns": 1}] - ).to_data() + metric_record = create_metric_records_data(results=[{"request_latency_ns": 1}]) assert strategy.supports(timing_stats) is True assert strategy.supports(metric_record) is False @@ -217,7 +215,7 @@ async def test_process_ignores_non_timing_inputs(self) -> None: context = _TimingStrategyContext(attributes={}) strategy = TimingResultsStrategy(context) - await strategy.process(create_metric_records_message(results=[]).to_data()) + await strategy.process(create_metric_records_data(results=[])) assert context.build_timing_attributes_calls == [] assert context.counter_delta_calls == [] diff --git a/tests/unit/property/_numeric_bounds_baseline.txt b/tests/unit/property/_numeric_bounds_baseline.txt index 69eff8d5f9..76d6c11669 100644 --- a/tests/unit/property/_numeric_bounds_baseline.txt +++ b/tests/unit/property/_numeric_bounds_baseline.txt @@ -39,7 +39,6 @@ AioHttpTraceData.response_receive_start_perf_ns: numeric field has no ge/gt/le/l AioHttpTraceData.response_status_code: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. AioHttpTraceData.tcp_connect_end_perf_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. AioHttpTraceData.tcp_connect_start_perf_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. -AccuracyRecordsMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. AllRecordsReceivedMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. AnalysisStats.total_requests: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. AnalysisStats.unique_prefixes: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. @@ -217,8 +216,6 @@ MetricRecordMetadata.request_start_ns: numeric field has no ge/gt/le/lt bound an MetricRecordMetadata.session_num: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. MetricRecordMetadata.turn_index: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. MetricRecordsData.metrics: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. -MetricRecordsMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. -MetricRecordsMessage.results: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. MetricResult.avg: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. MetricResult.count: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. MetricResult.current: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. @@ -302,6 +299,7 @@ RecordContext.agent_depth: numeric field has no ge/gt/le/lt bound and is not Fin RecordContext.audio_duration_seconds: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. RecordContext.max_tokens: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. RecordContext.turn_index: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. +RecordsMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. RecordsProcessingStatsMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. RegisterServiceCommand.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. RegistrationMessage.request_ns: numeric field has no ge/gt/le/lt bound and is not FiniteFloat. Add a Pydantic numeric constraint, annotate as FiniteFloat, or add to NUMERIC_BOUNDS_WHITELIST. diff --git a/tests/unit/records/test_realtime_metrics_generation.py b/tests/unit/records/test_realtime_metrics_generation.py index ad211d6d52..a16708b94b 100644 --- a/tests/unit/records/test_realtime_metrics_generation.py +++ b/tests/unit/records/test_realtime_metrics_generation.py @@ -19,7 +19,7 @@ class name so a stale dashboard/log block is diagnosable. from aiperf.records.records_manager_processing import generate_realtime_metrics from tests.unit.post_processors.conftest import ( create_accumulator_with_metrics, - create_metric_records_message, + create_metric_records_data, ) @@ -99,22 +99,22 @@ async def test_generate_realtime_metrics_excludes_warmup_records( no phase mask and averaged the warmup + profiling latencies together.""" accumulator = create_accumulator_with_metrics(benchmark_run, RequestLatencyMetric) - warmup = create_metric_records_message( + warmup = create_metric_records_data( session_num=0, benchmark_phase=CreditPhase.WARMUP, request_start_ns=1_000_000_000, request_end_ns=1_100_000_000, results=[{RequestLatencyMetric.tag: 100_000_000.0}], ) - profiling = create_metric_records_message( + profiling = create_metric_records_data( session_num=0, benchmark_phase=CreditPhase.PROFILING, request_start_ns=2_000_000_000, request_end_ns=2_200_000_000, results=[{RequestLatencyMetric.tag: 200_000_000.0}], ) - await accumulator.process_record(warmup.to_data()) - await accumulator.process_record(profiling.to_data()) + await accumulator.process_record(warmup) + await accumulator.process_record(profiling) flat = await generate_realtime_metrics([accumulator]) diff --git a/tests/unit/records/test_record_processor_service.py b/tests/unit/records/test_record_processor_service.py index 6c11ad0872..7d0bfa9742 100644 --- a/tests/unit/records/test_record_processor_service.py +++ b/tests/unit/records/test_record_processor_service.py @@ -9,12 +9,11 @@ from aiperf.accuracy.models import AccuracyRecordsData from aiperf.common.enums import CreditPhase, ExportLevel from aiperf.common.messages import ( - AccuracyRecordsMessage, BaseServiceErrorMessage, - MetricRecordsMessage, + MetricRecordsData, + RecordsMessage, ) from aiperf.common.utils import compute_time_ns -from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.records.record_processor_service import RecordProcessor from tests.unit.post_processors.conftest import create_metric_metadata @@ -36,77 +35,120 @@ def _make_accuracy_record(session_num: int = 0) -> AccuracyRecordsData: ) -class TestRecordProcessorTypedRecordPartition: - """Processor outputs are partitioned by transport: MetricRecordDict (a dict - subclass) stays in MetricRecordsMessage.results; typed Pydantic records - (AccuracyRecordsData) travel on their own dedicated channel message.""" +def _make_processor_mock(producers, observers=None): + """Build a RecordProcessor MagicMock wired for the 2-stage generic ship.""" + mock_self = MagicMock(spec=RecordProcessor) + mock_self.service_id = "rp" + mock_self.run = MagicMock() + mock_self.run.cfg.artifacts.export_level = ExportLevel.RECORDS + mock_self.records_push_client = AsyncMock() + mock_self.inference_result_parser = MagicMock() + mock_self.inference_result_parser.parse_request_record = AsyncMock( + return_value=MagicMock() + ) + mock_self._create_metric_record_metadata = MagicMock( + return_value=create_metric_metadata(session_num=0) + ) + mock_self._free_record_data = MagicMock(return_value=(None, None)) + mock_self._producers = producers + mock_self._observers = observers or [] + return mock_self + + +class TestRecordProcessorGenericShip: + """Stage 1 producers emit finished typed records grouped by their declared + record_type; stage 2 observers view them via a RecordObserverContext; then + exactly ONE generic RecordsMessage per inference record ships every produced + record on the envelope (no per-type message class, no builder map).""" @pytest.mark.asyncio - async def test_process_and_forward_splits_dicts_from_typed_records(self): - """A mixed result list pushes the dict in MetricRecordsMessage.results and - the accuracy record in a separate AccuracyRecordsMessage.""" - metric_dict = MetricRecordDict({"some_metric": 1.0}) + async def test_process_and_forward_ships_single_generic_message(self): + """A metric producer + an accuracy producer flatten into one RecordsMessage.""" + metric_data = MetricRecordsData( + metadata=create_metric_metadata(session_num=0), metrics={"some_metric": 1.0} + ) accuracy_record = _make_accuracy_record() - mock_self = MagicMock(spec=RecordProcessor) - mock_self.service_id = "rp" - mock_self.run = MagicMock() - mock_self.run.cfg.artifacts.export_level = ExportLevel.RECORDS - mock_self.records_push_client = AsyncMock() - mock_self.inference_result_parser = MagicMock() - mock_self.inference_result_parser.parse_request_record = AsyncMock( - return_value=MagicMock() - ) - mock_self._create_metric_record_metadata = MagicMock( - return_value=create_metric_metadata(session_num=0) - ) - mock_self._process_record = AsyncMock( - return_value=[metric_dict, accuracy_record] - ) - mock_self._free_record_data = MagicMock(return_value=(None, None)) - # Run the real partition/push seam against this mock instance. - mock_self._push_typed_records = ( - lambda recs: RecordProcessor._push_typed_records(mock_self, recs) + metric_producer = MagicMock() + metric_producer.process_record = AsyncMock(return_value=metric_data) + accuracy_producer = MagicMock() + accuracy_producer.process_record = AsyncMock(return_value=accuracy_record) + + mock_self = _make_processor_mock( + producers=[ + ("metric_records", metric_producer), + ("accuracy", accuracy_producer), + ] ) await RecordProcessor._process_and_forward_record( mock_self, MagicMock(service_id="w1"), MagicMock(), None ) - pushed = [c.args[0] for c in mock_self.records_push_client.push.await_args_list] - metric_msgs = [m for m in pushed if isinstance(m, MetricRecordsMessage)] - accuracy_msgs = [m for m in pushed if isinstance(m, AccuracyRecordsMessage)] + mock_self.records_push_client.push.assert_awaited_once() + msg = mock_self.records_push_client.push.await_args.args[0] + assert isinstance(msg, RecordsMessage) + assert metric_data in msg.records + assert accuracy_record in msg.records + assert len(msg.records) == 2 - assert len(metric_msgs) == 1 - assert metric_msgs[0].results == [metric_dict] - assert all(isinstance(r, dict) for r in metric_msgs[0].results) + @pytest.mark.asyncio + async def test_observers_receive_context_with_grouped_produced(self): + """Observers get a RecordObserverContext exposing ctx.metrics and + ctx.get(record_type), grouped by the producer's declared record_type.""" + metric_data = MetricRecordsData( + metadata=create_metric_metadata(session_num=0), metrics={"some_metric": 1.0} + ) + accuracy_record = _make_accuracy_record() - assert len(accuracy_msgs) == 1 - assert accuracy_msgs[0].records == [accuracy_record] + metric_producer = MagicMock() + metric_producer.process_record = AsyncMock(return_value=metric_data) + accuracy_producer = MagicMock() + accuracy_producer.process_record = AsyncMock(return_value=accuracy_record) - @pytest.mark.asyncio - async def test_push_typed_records_groups_by_record_type(self): - """All accuracy records land in a single AccuracyRecordsMessage.""" - mock_self = MagicMock(spec=RecordProcessor) - mock_self.service_id = "rp" - mock_self.records_push_client = AsyncMock() + observer = MagicMock() + observer.observe = AsyncMock() - records = [_make_accuracy_record(0), _make_accuracy_record(1)] - await RecordProcessor._push_typed_records(mock_self, records) + mock_self = _make_processor_mock( + producers=[ + ("metric_records", metric_producer), + ("accuracy", accuracy_producer), + ], + observers=[observer], + ) - mock_self.records_push_client.push.assert_awaited_once() - msg = mock_self.records_push_client.push.await_args.args[0] - assert isinstance(msg, AccuracyRecordsMessage) - assert msg.records == records + await RecordProcessor._process_and_forward_record( + mock_self, MagicMock(service_id="w1"), MagicMock(), None + ) + + observer.observe.assert_awaited_once() + ctx = observer.observe.await_args.args[0] + assert ctx.metrics is metric_data + assert ctx.get("accuracy") == [accuracy_record] @pytest.mark.asyncio - async def test_push_typed_records_empty_is_noop(self): - mock_self = MagicMock(spec=RecordProcessor) - mock_self.records_push_client = AsyncMock() + async def test_producer_exception_does_not_block_ship(self): + """A failing producer is logged and skipped; the record still ships.""" + accuracy_record = _make_accuracy_record() + bad_producer = MagicMock() + bad_producer.process_record = AsyncMock(side_effect=RuntimeError("boom")) + accuracy_producer = MagicMock() + accuracy_producer.process_record = AsyncMock(return_value=accuracy_record) + + mock_self = _make_processor_mock( + producers=[ + ("metric_records", bad_producer), + ("accuracy", accuracy_producer), + ] + ) - await RecordProcessor._push_typed_records(mock_self, []) + await RecordProcessor._process_and_forward_record( + mock_self, MagicMock(service_id="w1"), MagicMock(), None + ) - mock_self.records_push_client.push.assert_not_awaited() + mock_self.records_push_client.push.assert_awaited_once() + msg = mock_self.records_push_client.push.await_args.args[0] + assert msg.records == [accuracy_record] class TestRecordProcessorCreateMetricRecordMetadata: @@ -241,7 +283,8 @@ async def test_on_dataset_configured_sets_event(self): """_on_dataset_configured must release the barrier once processors are configured.""" mock_self = MagicMock(spec=RecordProcessor) mock_self._dataset_configured_event = asyncio.Event() - mock_self.records_processors = [] + mock_self._producers = [] + mock_self._observers = [] await RecordProcessor._on_dataset_configured(mock_self, MagicMock()) @@ -304,7 +347,7 @@ async def _raise_timeout(coro, *args, **kwargs): class TestRecordProcessorLockstepGuard: """The lockstep contract requires that every received inference result - forwards exactly one MetricRecordsMessage. The error-forward path itself + forwards exactly one RecordsMessage. The error-forward path itself must therefore never drop the record, even when metadata creation fails or the forward call raises -- otherwise the timeout-less RecordsManager completion barrier hangs the run at end-of-phase. @@ -334,7 +377,9 @@ async def test_forward_failed_record_metadata_creation_raises_still_pushes_error mock_self.records_push_client.push.assert_awaited_once() pushed = mock_self.records_push_client.push.await_args.args[0] - assert pushed.results == [] + assert isinstance(pushed, RecordsMessage) + assert len(pushed.records) == 1 + assert pushed.records[0].metrics == {} assert pushed.error is not None @pytest.mark.asyncio diff --git a/tests/unit/records/test_records_manager.py b/tests/unit/records/test_records_manager.py index 4d82859b25..37e1f897e3 100644 --- a/tests/unit/records/test_records_manager.py +++ b/tests/unit/records/test_records_manager.py @@ -13,7 +13,7 @@ from aiperf.common.messages import BaseServiceErrorMessage from aiperf.common.messages.inference_messages import ( MetricRecordsData, - MetricRecordsMessage, + RecordsMessage, ) from aiperf.common.messages.telemetry_messages import TelemetryRecordsMessage from aiperf.common.models import ( @@ -36,6 +36,7 @@ CreditPhaseStartMessage, CreditsCompleteMessage, ) +from aiperf.metrics.accumulator import MetricsAccumulator from aiperf.metrics.accumulator_models import AccumulatorMetricsSummary from aiperf.metrics.cache_reporting_hint import CACHE_REPORTING_HINT from aiperf.plugin.enums import AccumulatorType, TimingMode @@ -158,26 +159,28 @@ def _make_manager(self) -> RecordsManager: manager.is_enabled_for = MagicMock(return_value=False) manager._dataset_configured_event = asyncio.Event() manager._dataset_configured_event.set() - manager._maybe_hint_missing_cache_reporting = MagicMock() manager._records_tracker = MagicMock() manager._records_tracker.check_and_set_all_records_received_for_phase.return_value = False manager._error_tracker = ErrorTracker() manager._complete_credit_phases = set() return manager + def _records_message(self) -> RecordsMessage: + record = create_metric_record_data(1_000, 2_000) + return RecordsMessage( + service_id="rp", metadata=record.metadata, records=[record] + ) + @pytest.mark.asyncio async def test_metric_dispatch_error_recorded_in_phase_error_summary(self) -> None: manager = self._make_manager() dispatch_error = RuntimeError("metric accumulator failed") manager._dispatch_record = AsyncMock(return_value=[dispatch_error]) - message = MagicMock() - message.to_data.return_value = create_metric_record_data(1_000, 2_000) - - await manager._on_metric_records(message) + await manager._on_records(self._records_message()) # Record is still counted, but the handler failure is not swallowed. - manager._records_tracker.update_from_record_data.assert_called_once() + manager._records_tracker.update_from_request.assert_called_once() summary = manager._error_tracker.get_error_summary_for_phase( CreditPhase.PROFILING ) @@ -189,10 +192,7 @@ async def test_successful_metric_dispatch_records_no_phase_error(self) -> None: manager = self._make_manager() manager._dispatch_record = AsyncMock(return_value=[]) - message = MagicMock() - message.to_data.return_value = create_metric_record_data(1_000, 2_000) - - await manager._on_metric_records(message) + await manager._on_records(self._records_message()) assert ( manager._error_tracker.get_error_summary_for_phase(CreditPhase.PROFILING) @@ -295,7 +295,6 @@ def _create_manager_for_timing_dispatch() -> RecordsManager: manager._phase_branch_stats = {} manager._latest_branch_stats = None manager._dispatch_record = AsyncMock(return_value=[]) - manager._maybe_hint_missing_cache_reporting = MagicMock() manager.info = MagicMock() manager.notice = MagicMock() manager.debug = MagicMock() @@ -307,20 +306,25 @@ def _create_manager_for_timing_dispatch() -> RecordsManager: def _metric_records_message( phase: CreditPhase = CreditPhase.PROFILING, -) -> MetricRecordsMessage: - return MetricRecordsMessage( +) -> RecordsMessage: + metadata = MetricRecordMetadata( + session_num=17, + conversation_id="conv-2026-05-14-race", + turn_index=0, + request_start_ns=1_000_000_000, + request_end_ns=1_250_000_000, + worker_id="worker-a100-03", + record_processor_id="record-processor-rp-7f2a", + benchmark_phase=phase, + ) + return RecordsMessage( service_id="record-processor-rp-7f2a", - metadata=MetricRecordMetadata( - session_num=17, - conversation_id="conv-2026-05-14-race", - turn_index=0, - request_start_ns=1_000_000_000, - request_end_ns=1_250_000_000, - worker_id="worker-a100-03", - record_processor_id="record-processor-rp-7f2a", - benchmark_phase=phase, - ), - results=[{"request_latency": 250_000_000}], + metadata=metadata, + records=[ + MetricRecordsData( + metadata=metadata, metrics={"request_latency": 250_000_000} + ) + ], ) @@ -404,9 +408,9 @@ async def test_on_metric_records_records_complete_before_phase_complete_defers_f final_requests_completed=64, ) - await manager._on_metric_records(_metric_records_message()) + await manager._on_records(_metric_records_message()) - manager._records_tracker.update_from_record_data.assert_called_once() + manager._records_tracker.update_from_request.assert_called_once() manager._records_tracker.check_and_set_all_records_received_for_phase.assert_not_called() manager._handle_all_records_received.assert_not_awaited() @@ -497,7 +501,7 @@ async def _record_branch_stats_at_finalization(phase: CreditPhase) -> None: manager._records_tracker.check_and_set_all_records_received_for_phase.reset_mock() manager._records_tracker.check_and_set_all_records_received_for_phase.return_value = True - await manager._on_metric_records(_metric_records_message()) + await manager._on_records(_metric_records_message()) manager._records_tracker.check_and_set_all_records_received_for_phase.assert_called_once_with( CreditPhase.PROFILING @@ -539,7 +543,7 @@ async def test_finalization_runs_once_for_all_terminal_event_orders( elif event == "credits_complete": await manager._on_credits_complete(credits_complete) else: - await manager._on_metric_records(metric_record) + await manager._on_records(metric_record) manager._handle_all_records_received.assert_awaited_once_with( CreditPhase.PROFILING @@ -573,7 +577,7 @@ async def _block_timing_dispatch(record) -> list[BaseException]: ) await timing_dispatch_started.wait() - await manager._on_metric_records(_metric_records_message()) + await manager._on_records(_metric_records_message()) manager._handle_all_records_received.assert_not_awaited() release_timing_dispatch.set() @@ -594,9 +598,9 @@ async def test_dispatch_errors_still_update_tracker_and_converge_barrier( manager._complete_credit_phases = {CreditPhase.PROFILING} manager._records_tracker.check_and_set_all_records_received_for_phase.return_value = True - await manager._on_metric_records(_metric_records_message()) + await manager._on_records(_metric_records_message()) - manager._records_tracker.update_from_record_data.assert_called_once() + manager._records_tracker.update_from_request.assert_called_once() manager._records_tracker.check_and_set_all_records_received_for_phase.assert_called_once_with( CreditPhase.PROFILING ) @@ -689,34 +693,34 @@ async def test_completed_excludes_analyzer_metrics(self) -> None: class TestMidRunCacheReportingHint: - """RecordsManager warns once when usage lacks prompt-cache read tokens.""" + """MetricsAccumulator warns once when usage lacks prompt-cache read tokens.""" - def _manager(self) -> RecordsManager: - manager = RecordsManager.__new__(RecordsManager) - manager.warning = MagicMock() - manager._warned_missing_cache_reporting = False - return manager + def _accumulator(self) -> MetricsAccumulator: + accumulator = MetricsAccumulator.__new__(MetricsAccumulator) + accumulator.warning = MagicMock() + accumulator._warned_missing_cache_reporting = False + return accumulator def test_warns_once_on_first_qualifying_record(self) -> None: - manager = self._manager() + accumulator = self._accumulator() record_data = SimpleNamespace(metrics={"usage_prompt_tokens": 1024}) - manager._maybe_hint_missing_cache_reporting(record_data) - manager._maybe_hint_missing_cache_reporting(record_data) - manager.warning.assert_called_once_with(CACHE_REPORTING_HINT) + accumulator._maybe_hint_missing_cache_reporting(record_data) + accumulator._maybe_hint_missing_cache_reporting(record_data) + accumulator.warning.assert_called_once_with(CACHE_REPORTING_HINT) def test_no_warning_when_cache_reported(self) -> None: - manager = self._manager() + accumulator = self._accumulator() record_data = SimpleNamespace( metrics={"usage_prompt_tokens": 1024, "usage_prompt_cache_read_tokens": 0} ) - manager._maybe_hint_missing_cache_reporting(record_data) - manager.warning.assert_not_called() + accumulator._maybe_hint_missing_cache_reporting(record_data) + accumulator.warning.assert_not_called() def test_no_warning_when_usage_absent(self) -> None: - manager = self._manager() + accumulator = self._accumulator() record_data = SimpleNamespace(metrics={"output_sequence_length": 32}) - manager._maybe_hint_missing_cache_reporting(record_data) - manager.warning.assert_not_called() + accumulator._maybe_hint_missing_cache_reporting(record_data) + accumulator.warning.assert_not_called() class TestRealtimeUpdateGate: @@ -785,13 +789,12 @@ async def test_on_metric_records_waits_for_dataset_configured(self) -> None: manager._records_tracker = MagicMock() manager._error_tracker = MagicMock() manager._complete_credit_phases = set() - manager._maybe_hint_missing_cache_reporting = MagicMock() manager._dispatch_record = AsyncMock( side_effect=RuntimeError("REACHED_PROCESSING") ) message = _metric_records_message() - task = asyncio.create_task(manager._on_metric_records(message)) + task = asyncio.create_task(manager._on_records(message)) for _ in range(3): await asyncio.sleep(0) @@ -823,7 +826,7 @@ async def _raise_timeout(coro, *args, **kwargs): "aiperf.records.dataset_gate.asyncio.wait_for", _raise_timeout ) - await manager._on_metric_records(message) + await manager._on_records(message) manager._kill.assert_awaited_once() published = manager.publish.await_args.args[0] diff --git a/tests/unit/records/test_records_manager_accuracy.py b/tests/unit/records/test_records_manager_accuracy.py index 28835c67b9..866e9eca6a 100644 --- a/tests/unit/records/test_records_manager_accuracy.py +++ b/tests/unit/records/test_records_manager_accuracy.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +import asyncio from unittest.mock import AsyncMock, MagicMock import pytest @@ -9,9 +10,10 @@ from aiperf.accuracy.models import AccuracyRecordsData, AccuracySummary from aiperf.common.enums import CreditPhase from aiperf.common.messages import ( - AccuracyRecordsMessage, ProcessAccuracyResultMessage, + RecordsMessage, ) +from aiperf.common.models.record_models import MetricRecordMetadata from aiperf.records.records_manager import RecordsManager @@ -32,17 +34,46 @@ def _make_accuracy_record(session_num: int = 0) -> AccuracyRecordsData: ) +def _metadata() -> MetricRecordMetadata: + return MetricRecordMetadata( + session_num=0, + request_start_ns=1_000, + request_end_ns=2_000, + worker_id="w1", + record_processor_id="rp", + benchmark_phase=CreditPhase.PROFILING, + ) + + +def _records_manager_for_dispatch(dispatch_result: list) -> MagicMock: + mgr = MagicMock() + mgr.debug = MagicMock() + mgr.trace = MagicMock() + mgr.is_trace_enabled = False + mgr._dataset_configured_event = asyncio.Event() + mgr._dataset_configured_event.set() + mgr._records_tracker = MagicMock() + mgr._records_tracker.check_and_set_all_records_received_for_phase.return_value = ( + False + ) + mgr._error_tracker = MagicMock() + mgr._complete_credit_phases = set() + mgr._dispatch_record = AsyncMock(return_value=dispatch_result) + mgr._on_records = RecordsManager._on_records.__get__(mgr) + return mgr + + class TestOnAccuracyRecords: + """Accuracy records ride the generic RecordsMessage envelope and are + dispatched by ``_on_records`` without any accuracy-specific handler.""" + @pytest.mark.asyncio async def test_dispatches_each_record(self) -> None: - mgr = MagicMock() - mgr.debug = MagicMock() - mgr._dispatch_record = AsyncMock(return_value=[]) - mgr._on_accuracy_records = RecordsManager._on_accuracy_records.__get__(mgr) + mgr = _records_manager_for_dispatch([]) records = [_make_accuracy_record(0), _make_accuracy_record(1)] - await mgr._on_accuracy_records( - AccuracyRecordsMessage(service_id="rp", records=records) + await mgr._on_records( + RecordsMessage(service_id="rp", metadata=_metadata(), records=records) ) assert mgr._dispatch_record.await_count == 2 @@ -50,17 +81,16 @@ async def test_dispatches_each_record(self) -> None: assert dispatched == records @pytest.mark.asyncio - async def test_dispatch_errors_are_logged_not_raised(self) -> None: - mgr = MagicMock() - mgr.debug = MagicMock() - mgr._dispatch_record = AsyncMock(return_value=[ValueError("boom")]) - mgr._on_accuracy_records = RecordsManager._on_accuracy_records.__get__(mgr) + async def test_dispatch_errors_are_tracked_not_raised(self) -> None: + mgr = _records_manager_for_dispatch([ValueError("boom")]) - await mgr._on_accuracy_records( - AccuracyRecordsMessage(service_id="rp", records=[_make_accuracy_record()]) + await mgr._on_records( + RecordsMessage( + service_id="rp", metadata=_metadata(), records=[_make_accuracy_record()] + ) ) - assert mgr.debug.called + assert mgr._error_tracker.increment_error_count_for_phase.called class TestPublishAccuracyResults: From 2aa4e92cfeb197e3c2050914e378da10ac6786ce Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 23:06:19 -0700 Subject: [PATCH 28/39] refactor(records): serialized record_type discriminator for wire reconstruction Fix a runtime bug the unit tests missed: RecordsMessage.records typed as list[Any] serialized the typed records to dicts but could not reconstruct them on the ZMQ receiving side, so RecordsManager saw bare dicts and failed to route them ("no record_type"), aborting the run. Give the two channel records a SERIALIZED record_type discriminator via a shared RecordData(AIPerfBaseModel) base (discriminator_field ClassVar + record_type field). MetricRecordsData/AccuracyRecordsData now set record_type as a Literal field (not a ClassVar), so AutoRoutedModel auto-registers them and reconstructs the concrete subclass. RecordsMessage types records as list[SerializeAsAny[RecordData]] with a before-validator that routes each dict through RecordData.from_json (mirrors the existing BaseTraceData/trace_type pattern). Class-level MetricRecordsData.record_type access in RecordObserverContext now reads the field default. Exclude the wire-only record_type from accuracy_export.jsonl via a new mixin hook so the on-disk output stays byte-identical. Adds a wire encode/decode round-trip test asserting concrete types (incl. nested trace_data) survive. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/accuracy/jsonl_writer.py | 13 ++-- src/aiperf/accuracy/models.py | 15 +++-- .../common/messages/inference_messages.py | 34 +++++++--- .../mixins/buffered_jsonl_writer_mixin.py | 16 ++++- src/aiperf/common/models/__init__.py | 2 + src/aiperf/common/models/record_models.py | 23 ++++++- .../record_observer_context.py | 8 ++- tests/unit/accuracy/test_accuracy_models.py | 67 ++++++++++++++++++- 8 files changed, 151 insertions(+), 27 deletions(-) diff --git a/src/aiperf/accuracy/jsonl_writer.py b/src/aiperf/accuracy/jsonl_writer.py index d00419225c..ce62e1b5d8 100644 --- a/src/aiperf/accuracy/jsonl_writer.py +++ b/src/aiperf/accuracy/jsonl_writer.py @@ -21,12 +21,17 @@ class AccuracyJSONLWriter( ): """Exports per-record graded accuracy data to JSONL files. - Streams each ``AccuracyRecordsData`` as it arrives on the dedicated accuracy - channel, writing one JSON line per graded response. Each line carries the - grade (pass/unparsed/confidence), the expected/actual answers, and the - grader's reasoning, enabling per-response post-hoc analysis. + Streams each ``AccuracyRecordsData`` as it arrives, writing one JSON line per + graded response. Each line carries the grade (pass/unparsed/confidence), the + expected/actual answers, and the grader's reasoning, enabling per-response + post-hoc analysis. """ + # ``record_type`` is a wire-only discriminator (needed to reconstruct the + # record across the ZMQ boundary on the generic RecordsMessage) -- exclude it + # from accuracy_export.jsonl so the on-disk output is byte-identical. + _jsonl_exclude_fields = {"record_type"} + def __init__( self, run: BenchmarkRun, diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index 15218fa090..22193ab468 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -3,12 +3,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Literal +from typing import TYPE_CHECKING, Any, Literal from pydantic import Field from typing_extensions import TypedDict from aiperf.common.enums import CreditPhase +from aiperf.common.models import RecordData from aiperf.common.models.base_models import AIPerfBaseModel if TYPE_CHECKING: @@ -82,15 +83,15 @@ class BenchmarkProblem(AIPerfBaseModel): ) -class AccuracyRecordsData(AIPerfBaseModel): - """Per-graded-response record that flows on the dedicated ``accuracy`` channel. +class AccuracyRecordsData(RecordData): + """Per-graded-response record that rides the generic ``RecordsMessage`` envelope. - Mirrors the record-type pattern used by ``ServerMetricsRecord`` and - ``TelemetryRecord``: ``record_type`` is a plain ``ClassVar`` (not a Pydantic - field) read by the routing layer via ``getattr(record, "record_type")``. + ``record_type`` is a SERIALIZED ``Literal`` discriminator field (not a + ClassVar) so AutoRoutedModel reconstructs the concrete type across the ZMQ + boundary; the routing layer still reads it via ``getattr(record, "record_type")``. """ - record_type: ClassVar[str] = "accuracy" + record_type: Literal["accuracy"] = "accuracy" session_num: int = Field( ge=0, diff --git a/src/aiperf/common/messages/inference_messages.py b/src/aiperf/common/messages/inference_messages.py index 67657d40ae..a6c75b17e6 100644 --- a/src/aiperf/common/messages/inference_messages.py +++ b/src/aiperf/common/messages/inference_messages.py @@ -1,14 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Any, ClassVar +from typing import Any, Literal from pydantic import Field, SerializeAsAny, field_validator from aiperf.common.enums import MessageType, MetricValueTypeT from aiperf.common.messages.service_messages import BaseServiceMessage -from aiperf.common.models import ErrorDetails, RequestRecord -from aiperf.common.models.base_models import AIPerfBaseModel +from aiperf.common.models import ErrorDetails, RecordData, RequestRecord from aiperf.common.models.record_models import MetricRecordMetadata, MetricResult from aiperf.common.models.trace_models import BaseTraceData from aiperf.common.types import MessageTypeT, MetricTagT @@ -24,10 +23,10 @@ class InferenceResultsMessage(BaseServiceMessage): ) -class MetricRecordsData(AIPerfBaseModel): +class MetricRecordsData(RecordData): """Incoming data from the record processor service to combine metric records for the profile run.""" - record_type: ClassVar[str] = "metric_records" + record_type: Literal["metric_records"] = "metric_records" metadata: MetricRecordMetadata = Field( ..., description="The metadata of the request record." @@ -65,8 +64,8 @@ class RecordsMessage(BaseServiceMessage): One ``RecordsMessage`` is pushed for every inference record (the credit lockstep contract). It carries the request metadata plus the flattened list of typed records produced for that request. Each record self-identifies via - its own ``record_type`` ClassVar, so the records manager dispatches - generically without inspecting the message for a record type. + its own serialized ``record_type`` discriminator, so the records manager + dispatches generically without inspecting the message for a record type. """ message_type: MessageTypeT = MessageType.RECORDS @@ -74,16 +73,33 @@ class RecordsMessage(BaseServiceMessage): metadata: MetricRecordMetadata = Field( ..., description="The metadata of the request record; drives the lockstep." ) - records: list[SerializeAsAny[Any]] = Field( + records: list[SerializeAsAny[RecordData]] = Field( default_factory=list, description="The typed records produced for this request. Each record " - "self-identifies via its own record_type ClassVar.", + "self-identifies via its own serialized record_type discriminator, so the " + "concrete subclass is reconstructed on the receiving side of the ZMQ boundary.", ) error: ErrorDetails | None = Field( default=None, description="The request-level error details if the request failed.", ) + @field_validator("records", mode="before") + @classmethod + def route_records(cls, v: Any) -> Any: + """Reconstruct each dict record into its concrete RecordData subclass. + + On the wire the records arrive as plain dicts; route each one via + ``RecordData.from_json`` (using the serialized ``record_type`` + discriminator) so the records manager receives concrete typed records, + not bare dicts. Already-constructed model instances pass through. + """ + if not isinstance(v, list): + return v + return [ + RecordData.from_json(item) if isinstance(item, dict) else item for item in v + ] + @property def valid(self) -> bool: """Whether the request was valid.""" diff --git a/src/aiperf/common/mixins/buffered_jsonl_writer_mixin.py b/src/aiperf/common/mixins/buffered_jsonl_writer_mixin.py index fcbbe18646..7301454d0a 100644 --- a/src/aiperf/common/mixins/buffered_jsonl_writer_mixin.py +++ b/src/aiperf/common/mixins/buffered_jsonl_writer_mixin.py @@ -6,7 +6,7 @@ import contextlib import time from pathlib import Path -from typing import Generic +from typing import ClassVar, Generic import aiofiles import orjson @@ -34,6 +34,12 @@ class BufferedJSONLWriterMixin(AIPerfLifecycleMixin, Generic[BaseModelT]): lines_written: Number of lines written """ + # Field names to exclude from each serialized JSONL line. Subclasses that + # write a model carrying a wire-only field (e.g. a RecordData ``record_type`` + # discriminator needed for ZMQ reconstruction but not wanted on disk) set this + # to keep the on-disk output identical to before that field was added. + _jsonl_exclude_fields: ClassVar[set[str] | None] = None + def __init__( self, output_file: Path, @@ -112,7 +118,13 @@ async def buffered_write(self, record: BaseModelT) -> None: # scrub_non_finite enforces "null on disk = absent" across the # JSONL so per-record NaN/inf doesn't masquerade as missing. json_bytes = orjson.dumps( - scrub_non_finite(record.model_dump(exclude_none=True, mode="json")) + scrub_non_finite( + record.model_dump( + exclude_none=True, + mode="json", + exclude=self._jsonl_exclude_fields, + ) + ) ) buffer_to_flush = None diff --git a/src/aiperf/common/models/__init__.py b/src/aiperf/common/models/__init__.py index bb347ea600..92163ef0f8 100644 --- a/src/aiperf/common/models/__init__.py +++ b/src/aiperf/common/models/__init__.py @@ -84,6 +84,7 @@ RawRecordInfo, ReasoningResponseData, RecordContext, + RecordData, RequestInfo, RequestRecord, SSEField, @@ -233,6 +234,7 @@ "RawRecordInfo", "ReasoningResponseData", "RecordContext", + "RecordData", "RequestInfo", "RequestRecord", "SSEField", diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index 52eafd989c..7470a57c2f 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -7,7 +7,7 @@ import time from dataclasses import dataclass, field from functools import cached_property -from typing import Annotated, Any, AnyStr, Protocol, runtime_checkable +from typing import Annotated, Any, AnyStr, ClassVar, Protocol, runtime_checkable import orjson from pydantic import ( @@ -123,6 +123,27 @@ def to_json_result(self) -> JsonMetricResult: return result +class RecordData(AIPerfBaseModel): + """Base for typed records that travel on the generic ``RecordsMessage`` envelope. + + Subclasses declare a SERIALIZED ``record_type`` discriminator (a ``Literal`` + field, not a ClassVar) so AutoRoutedModel can reconstruct the concrete type + on the receiving side of the ZMQ boundary. A ClassVar discriminator would not + serialize, so the receiver would see bare dicts and fail to route them. This + mirrors the ``BaseTraceData`` / ``trace_type`` discriminated-union pattern. + + ``RecordData.from_json(dict)`` routes to the registered subclass by its + ``record_type`` value; ``getattr(instance, "record_type")`` returns the field + value, so the records-manager routing table keys off instances unchanged. + """ + + discriminator_field: ClassVar[str] = "record_type" + + record_type: str = Field( + description="Discriminator: the record_type channel this record routes on.", + ) + + class MetricValue(AIPerfBaseModel): """The value of a metric converted to display units for export.""" diff --git a/src/aiperf/post_processors/record_observer_context.py b/src/aiperf/post_processors/record_observer_context.py index 8223798aed..d91e3c5f94 100644 --- a/src/aiperf/post_processors/record_observer_context.py +++ b/src/aiperf/post_processors/record_observer_context.py @@ -12,6 +12,12 @@ if TYPE_CHECKING: from aiperf.common.models import MetricRecordMetadata, ParsedResponseRecord +# The record_type channel the metric producer declares in plugins.yaml, which is +# also the key the RecordProcessorService groups its output under in ``produced``. +# ``record_type`` is now a serialized field, so class access returns FieldInfo -- +# read the Literal default instead of ``MetricRecordsData.record_type``. +_METRIC_RECORDS_TYPE: str = MetricRecordsData.model_fields["record_type"].default + @dataclass(slots=True) class RecordObserverContext: @@ -36,5 +42,5 @@ def get(self, record_type: str) -> list[Any]: @property def metrics(self) -> MetricRecordsData | None: """The first metric-records output, or None when no producer emitted one.""" - items = self.produced.get(MetricRecordsData.record_type) or [] + items = self.produced.get(_METRIC_RECORDS_TYPE) or [] return items[0] if items else None diff --git a/tests/unit/accuracy/test_accuracy_models.py b/tests/unit/accuracy/test_accuracy_models.py index 8ea9b650fa..5072256a3c 100644 --- a/tests/unit/accuracy/test_accuracy_models.py +++ b/tests/unit/accuracy/test_accuracy_models.py @@ -11,10 +11,13 @@ ) from aiperf.common.enums import CreditPhase, MessageType from aiperf.common.messages import ( + Message, + MetricRecordsData, ProcessAccuracyResultMessage, RecordsMessage, ) from aiperf.common.models.record_models import MetricRecordMetadata +from aiperf.common.models.trace_models import AioHttpTraceData def _make_record(**overrides) -> AccuracyRecordsData: @@ -39,11 +42,12 @@ def _make_record(**overrides) -> AccuracyRecordsData: def test_accuracy_record_type_is_classvar_constant() -> None: a = _make_record(session_num=1) b = _make_record(session_num=2) - assert AccuracyRecordsData.record_type == "accuracy" + assert AccuracyRecordsData.model_fields["record_type"].default == "accuracy" assert a.record_type == "accuracy" assert b.record_type == "accuracy" - # record_type is a ClassVar, not a constructor field: it must not be dumped. - assert "record_type" not in a.model_dump() + # record_type is now a SERIALIZED discriminator field (needed for wire + # reconstruction on the generic RecordsMessage), so it IS dumped. + assert a.model_dump()["record_type"] == "accuracy" def test_accuracy_record_task_defaults_to_none() -> None: @@ -124,6 +128,63 @@ def test_records_message_serializes_accuracy_records() -> None: dumped = message.model_dump() assert len(dumped["records"]) == 2 assert dumped["records"][0]["session_num"] == 1 + assert dumped["records"][0]["record_type"] == "accuracy" + + +def test_records_message_wire_roundtrip_reconstructs_concrete_types() -> None: + """The generic RecordsMessage must survive the ZMQ encode/decode boundary + with each heterogeneous record rebuilt as its CONCRETE RecordData subclass. + + This is the regression guard for the serialization bug: a ``list[Any]`` + would decode the records back into bare dicts (no record_type routing), so + the records manager would fail to dispatch them. Serializing via the real + ``to_json_bytes`` / ``Message.from_json`` path proves the discriminator + reconstruction works. + """ + metadata = MetricRecordMetadata( + session_num=7, + request_start_ns=1_000, + request_end_ns=2_000, + worker_id="worker-1", + record_processor_id="rp", + benchmark_phase=CreditPhase.PROFILING, + ) + trace = AioHttpTraceData( + trace_type="aiohttp", + reference_time_ns=1_700_000_000_000_000_000, + reference_perf_ns=1_000_000_000, + request_send_start_perf_ns=1_000_000_000, + ) + metric_record = MetricRecordsData( + metadata=metadata, + metrics={"request_latency": 250_000_000}, + trace_data=trace, + ) + accuracy_record = _make_record(session_num=7) + + message = RecordsMessage( + service_id="record-processor", + metadata=metadata, + records=[metric_record, accuracy_record], + error=None, + ) + + decoded = Message.from_json(message.to_json_bytes()) + + assert isinstance(decoded, RecordsMessage) + assert len(decoded.records) == 2 + + round_metric, round_accuracy = decoded.records + assert isinstance(round_metric, MetricRecordsData) + assert round_metric.record_type == "metric_records" + assert round_metric.metrics["request_latency"] == 250_000_000 + # trace_data must survive with its concrete discriminated subtype intact. + assert isinstance(round_metric.trace_data, AioHttpTraceData) + assert round_metric.trace_data.trace_type == "aiohttp" + + assert isinstance(round_accuracy, AccuracyRecordsData) + assert round_accuracy.record_type == "accuracy" + assert round_accuracy.session_num == 7 def test_process_accuracy_result_message_carries_summary() -> None: From 72fc55aaf5f44a9b1f09a6748702937aea6db291 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 15 Jul 2026 23:14:47 -0700 Subject: [PATCH 29/39] refactor(records): drop dead message enums + document generic pipeline Follow-up cleanup after the producer/observer + generic RecordsMessage refactor: remove the now-unused MessageType.METRIC_RECORDS / ACCURACY_RECORD values (their handlers and message classes are gone), and update docs/architecture.md to describe the 2-stage producer/observer split and the single generic RecordsMessage (the deleted per-type message classes are no longer referenced). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/architecture.md | 8 +++++--- src/aiperf/common/enums/enums.py | 2 -- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 801e657d6c..fa8917aba1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -122,12 +122,14 @@ Accuracy benchmarking is one such dedicated channel, joining `metric_records`, ` | Channel (`record_type`) | Producer | Message | Consumers (`record_types`) | |---|---|---|---| -| `metric_records` | Record Processor | `MetricRecordsMessage` | `MetricsAccumulator`, JSONL/OTel stream exporters | +| `metric_records` | Record Processor (`MetricRecordProcessor`) | `RecordsMessage` | `MetricsAccumulator`, JSONL/OTel stream exporters | +| `accuracy` | Record Processor (`AccuracyRecordProcessor`) | `RecordsMessage` | `AccuracyAccumulator`, `AccuracyJSONLWriter` | | `gpu_telemetry` | GPU Telemetry Manager | `TelemetryRecordsMessage` | `GPUTelemetryAccumulator`, GPU JSONL writer | | `server_metrics` | Server Metrics Manager | `ServerMetricsRecordMessage` | `ServerMetricsAccumulator`, server-metrics JSONL writer | -| `accuracy` | Record Processor (`AccuracyRecordProcessor`) | `AccuracyRecordsMessage` | `AccuracyAccumulator`, `AccuracyJSONLWriter` | -The `AccuracyRecordProcessor` grades each parsed response against ground truth and returns an `AccuracyRecordsData` (`session_num`, `conversation_id`, `x_request_id`, `worker_id`, `benchmark_phase`, `timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, `confidence`, `expected`, `actual`, `explanation`, `model_output`, `model_thinking`). The `RecordProcessorService` partitions these typed records off the metric-dict stream and pushes an `AccuracyRecordsMessage` to the Records Manager, which fans each record out to the `AccuracyAccumulator` (rolls records into an `AccuracySummary` of overall + per-task pass/unparsed rates) and the `AccuracyJSONLWriter` (streams the full per-response grading detail to `accuracy_export.jsonl`, one JSON object per graded response). +**Two-stage record processing.** The `RecordProcessorService` runs two roles per parsed response (mirroring the accumulator/stream-exporter split on the consumer side): **producers** (`record_processor` category) parse and emit one finished typed record on a declared `record_type` channel — `MetricRecordProcessor` → `MetricRecordsData` (`metric_records`), `AccuracyRecordProcessor` → `AccuracyRecordsData` (`accuracy`) — and **observers** (`record_observer` category, e.g. the raw-record and outputs-json writers) receive a `RecordObserverContext` (the parsed record + all producer outputs keyed by `record_type`) and act without emitting a channel record. Each producer's output carries its own serialized `record_type` discriminator, so the service ships **one generic `RecordsMessage`** per request — `{metadata, records: list[RecordData], error}` — and the `RecordsManager` dispatches each record to its channel handlers by `record.record_type` (no per-type message classes, no type-sniffing). The metric record is the canonical per-request record that drives the phase-completion lockstep, keyed off the message metadata. + +The `AccuracyRecordProcessor` grades each parsed response against ground truth and returns an `AccuracyRecordsData` (`session_num`, `conversation_id`, `x_request_id`, `worker_id`, `benchmark_phase`, `timestamp_ns`, `task`, `grader_name`, `passed`, `unparsed`, `confidence`, `expected`, `actual`, `explanation`, `model_output`, `model_thinking`). The Records Manager fans each accuracy record out to the `AccuracyAccumulator` (rolls records into an `AccuracySummary` of overall + per-task pass/unparsed rates) and the `AccuracyJSONLWriter` (streams the full per-response grading detail to `accuracy_export.jsonl`, one JSON object per graded response). At end of the profiling phase, `RecordsManager._publish_accuracy_results(phase)` exports the phase-scoped `AccuracySummary` and publishes a `ProcessAccuracyResultMessage`. The `SystemController` receives it and stores the summary — gated in shutdown like the telemetry and server-metrics results. At export time, `SystemController._inject_accuracy_results_into_records` converts that summary back into legacy `accuracy.*` `MetricResult`s (via `AccuracySummary.to_metric_results()`) and appends them to `ProfileResults.records`. The `AccuracyConsoleExporter` and `AccuracyDataExporter` — like the main perf CSV/JSON exporters — then read those injected `accuracy.*` records. This inject bridge is what keeps the exported files (`accuracy_results.csv`, `profile_export_aiperf.{csv,json}`) byte-identical to the pre-refactor output. The dedicated per-response `accuracy_export.jsonl` is produced independently by the `AccuracyJSONLWriter` and is unaffected by this bridge. diff --git a/src/aiperf/common/enums/enums.py b/src/aiperf/common/enums/enums.py index c292ef6914..fe90a64667 100644 --- a/src/aiperf/common/enums/enums.py +++ b/src/aiperf/common/enums/enums.py @@ -362,7 +362,6 @@ class MessageType(CaseInsensitiveStrEnum): ERROR = "error" HEARTBEAT = "heartbeat" INFERENCE_RESULTS = "inference_results" - METRIC_RECORDS = "metric_records" RECORDS = "records" PARSED_INFERENCE_RESULTS = "parsed_inference_results" PROCESSING_STATS = "processing_stats" @@ -370,7 +369,6 @@ class MessageType(CaseInsensitiveStrEnum): PROCESS_TELEMETRY_RESULT = "process_telemetry_result" PROCESS_SERVER_METRICS_RESULT = "process_server_metrics_result" PROCESS_ACCURACY_RESULT = "process_accuracy_result" - ACCURACY_RECORD = "accuracy_record" PROCESS_ALL_RESULTS = "process_all_results" PROFILE_PROGRESS = "profile_progress" PROFILE_RESULTS = "profile_results" From 347f0c726e5dee0435c20d52ec56fddd16834427 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 16 Jul 2026 00:16:29 -0700 Subject: [PATCH 30/39] fix(records/accuracy): address code-review findings on the accumulator refactor Fixes validated findings from the PR #1144 branch review: - accuracy JSONL: skip warmup grades so the per-record export matches the phase-scoped AccuracySummary (was writing warmup + profiling while the summary counted profiling only). - cancel path: SystemController now bounded-waits for the accuracy result message before stop() so a Ctrl+C'd accuracy run still exports accuracy.* (new AIPERF_ACCURACY_CANCEL_RESULT_WAIT_SEC, default 5s). - accuracy publish: guard the publish and log at error (no longer silently swallowed by the caller); correct the overstated "exactly once" docstring. - routing: warn once (was debug-only) when a record_type has no handler, since such records are dropped while the request still counts as a success. - protocols: ServerMetrics/GPUTelemetry accumulator export_results signatures now match the ExportContext impls (async + ctx). - accuracy model_output: include tool_call_text so it matches the graded text. - energy analyzer: don't emit average_gpu_power=0 on a degenerate window. - worker_id field description corrected; drop dead _metric_state field; refresh plugin schema record_type/required_* value lists. Note: docs/cli-options.md regenerated to clear pre-existing generator drift the generate-cli-docs pre-commit hook requires. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/cli-options.md | 2 +- docs/environment-variables.md | 1 + .../accuracy/accuracy_record_processor.py | 16 ++++--- src/aiperf/accuracy/jsonl_writer.py | 9 +++- src/aiperf/accuracy/models.py | 4 +- src/aiperf/common/environment.py | 12 +++++ src/aiperf/controller/system_controller.py | 34 ++++++++++++++ src/aiperf/gpu_telemetry/protocols.py | 11 ++--- .../metrics/energy_efficiency_analyzer.py | 6 ++- src/aiperf/plugin/schema/plugins.schema.json | 6 +-- src/aiperf/plugin/schema/schemas.py | 10 +++-- src/aiperf/records/records_manager.py | 45 +++++++++++++------ src/aiperf/server_metrics/protocols.py | 14 ++---- .../records/test_records_manager_routing.py | 8 +++- 14 files changed, 126 insertions(+), 52 deletions(-) diff --git a/docs/cli-options.md b/docs/cli-options.md index 24700c3c12..758c70d102 100644 --- a/docs/cli-options.md +++ b/docs/cli-options.md @@ -1675,7 +1675,7 @@ Explore AIPerf plugins: aiperf plugins [category] [type] #### `--category` `` Category to explore. -
_Choices: [`accumulator`, `accuracy_benchmark`, `accuracy_grader`, `analyzer`, `api_router`, `arrival_pattern`, `communication`, `communication_client`, `console_exporter`, `convergence_criterion`, `custom_dataset_loader`, `data_exporter`, `dataset_backing_store`, `dataset_client_store`, `dataset_composer`, `dataset_sampler`, `endpoint`, `gpu_telemetry_collector`, `plot`, `public_dataset_loader`, `ramp`, `record_processor`, `search_planner`, `search_recipe`, `search_recipe_post_process`, `service`, `service_manager`, `stream_exporter`, `timing_strategy`, `transport`, `ui`, `url_selection_strategy`, `zmq_proxy`]_ +
_Choices: [`accumulator`, `accuracy_benchmark`, `accuracy_grader`, `analyzer`, `api_router`, `arrival_pattern`, `communication`, `communication_client`, `console_exporter`, `convergence_criterion`, `custom_dataset_loader`, `data_exporter`, `dataset_backing_store`, `dataset_client_store`, `dataset_composer`, `dataset_sampler`, `endpoint`, `gpu_telemetry_collector`, `plot`, `public_dataset_loader`, `ramp`, `record_observer`, `record_processor`, `search_planner`, `search_recipe`, `search_recipe_post_process`, `service`, `service_manager`, `stream_exporter`, `timing_strategy`, `transport`, `ui`, `url_selection_strategy`, `zmq_proxy`]_ #### `--name` `` diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 5ab56ae085..c37aac0a0d 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -36,6 +36,7 @@ Accuracy benchmark settings. Tunables for the accuracy benchmark loaders. Curren | Environment Variable | Default | Constraints | Description | |----------------------|---------|-------------|-------------| +| `AIPERF_ACCURACY_CANCEL_RESULT_WAIT_SEC` | `5.0` | ≥ 0.0 | Bounded time (seconds) the SystemController waits on the cancel (Ctrl+C) path for the RecordsManager's ProcessAccuracyResultMessage before stopping. The normal completion path blocks on the accuracy shutdown gate indefinitely, but the cancel path must not hang forever, so it waits at most this long for the graded accuracy summary to arrive over pub/sub before proceeding to export. Set to 0 to skip the wait entirely. | | `AIPERF_ACCURACY_LCB_RELEASE_TAG` | `'v4_v5'` | — | LiveCodeBench dataset subset (HF config name) passed as the positional ``name`` arg to ``load_dataset("livecodebench/code_generation_lite", name, split="test", trust_remote_code=True)``. Pins which monthly snapshot LCB serves so accuracy numbers are reproducible across runs and branches. Default ``v4_v5`` matches lighteval's base subset; bump (e.g. to ``v6``) when the team rebaselines against a newer snapshot. The loader always passes ``trust_remote_code=True`` so LCB's dataset-loading script can execute on ``datasets`` v4+ (mirrors lighteval's reference opt-in). Consumed by ``aiperf.accuracy.benchmarks.lcb_codegeneration``. | ## APISERVER diff --git a/src/aiperf/accuracy/accuracy_record_processor.py b/src/aiperf/accuracy/accuracy_record_processor.py index 373c1872da..001d763fe1 100644 --- a/src/aiperf/accuracy/accuracy_record_processor.py +++ b/src/aiperf/accuracy/accuracy_record_processor.py @@ -182,10 +182,12 @@ def _extract_output_and_thinking( ) -> tuple[str, str | None]: """Split the response into visible answer content and reasoning/thinking. - ``model_output`` is the answer channel (``TextResponseData.text`` or - ``ReasoningResponseData.content``); ``model_thinking`` is the concatenated + ``model_output`` is the answer channel (``TextResponseData.text``, + ``ReasoningResponseData.content``, or ``ToolCallResponseData`` content + + ``tool_call_text``); ``model_thinking`` is the concatenated ``reasoning_content`` from any ``ReasoningResponseData`` chunks, or None - when the model emitted no separate reasoning channel. + when the model emitted no separate reasoning channel. The output channel + mirrors what grading scores (``get_text()``) so the two never diverge. """ output_parts: list[str] = [] thinking_parts: list[str] = [] @@ -197,10 +199,14 @@ def _extract_output_and_thinking( if reasoning: thinking_parts.append(reasoning) content = getattr(data, "content", None) + tool_call_text = getattr(data, "tool_call_text", None) if content is not None: output_parts.append(content) - elif reasoning is None: - # Plain text (or tool-call) data with no reasoning channel. + if tool_call_text: + # Tool-call responses grade on content + tool_call_text; keep both. + output_parts.append(tool_call_text) + elif content is None and reasoning is None: + # Plain text data with no reasoning/tool-call channel. output_parts.append(data.get_text()) thinking = "".join(thinking_parts) if thinking_parts else None return "".join(output_parts), thinking diff --git a/src/aiperf/accuracy/jsonl_writer.py b/src/aiperf/accuracy/jsonl_writer.py index ce62e1b5d8..90ed282001 100644 --- a/src/aiperf/accuracy/jsonl_writer.py +++ b/src/aiperf/accuracy/jsonl_writer.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING from aiperf.accuracy.models import AccuracyRecordsData +from aiperf.common.enums import CreditPhase from aiperf.common.environment import Environment from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.mixins import BufferedJSONLWriterMixin @@ -54,7 +55,13 @@ def __init__( self.info(f"Accuracy JSONL export enabled: {self.output_file}") async def process_record(self, record: AccuracyRecordsData) -> None: - """Write a single graded accuracy record to the JSONL buffer.""" + """Write a single graded accuracy record to the JSONL buffer. + + Warmup grades are skipped so the per-record export stays consistent with + the phase-scoped ``AccuracySummary`` (which counts profiling only). + """ + if record.benchmark_phase != CreditPhase.PROFILING: + return await self.buffered_write(record) async def finalize(self) -> None: diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index 22193ab468..07ca452d5f 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -107,9 +107,7 @@ class AccuracyRecordsData(RecordData): description="Unique per-request id (X-Request-ID) for tracing this exact " "graded response back to the raw records", ) - worker_id: str = Field( - description="ID of the record processor that produced this record" - ) + worker_id: str = Field(description="ID of the worker that produced this record") benchmark_phase: CreditPhase = Field( description="Benchmark phase active when grading completed (warmup vs profiling)" ) diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index c240471616..cd904480c9 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -71,6 +71,18 @@ class _AccuracySettings(BaseSettings): model_config = SettingsConfigDict(env_prefix="AIPERF_ACCURACY_") + CANCEL_RESULT_WAIT_SEC: float = Field( + default=5.0, + ge=0.0, + description="Bounded time (seconds) the SystemController waits on the " + "cancel (Ctrl+C) path for the RecordsManager's " + "ProcessAccuracyResultMessage before stopping. The normal completion " + "path blocks on the accuracy shutdown gate indefinitely, but the cancel " + "path must not hang forever, so it waits at most this long for the " + "graded accuracy summary to arrive over pub/sub before proceeding to " + "export. Set to 0 to skip the wait entirely.", + ) + LCB_RELEASE_TAG: str = Field( default="v4_v5", description="LiveCodeBench dataset subset (HF config name) " diff --git a/src/aiperf/controller/system_controller.py b/src/aiperf/controller/system_controller.py index c1cf41d545..ee3f0f721b 100644 --- a/src/aiperf/controller/system_controller.py +++ b/src/aiperf/controller/system_controller.py @@ -981,11 +981,45 @@ async def _cancel_profiling(self) -> None: # Catch ANY exception during cancellation - we must always proceed to stop(). self.warning(f"Exception during cancel command (proceeding to stop): {e!r}") + # The normal completion path holds stop() on the accuracy shutdown gate + # until RecordsManager publishes ProcessAccuracyResultMessage; the cancel + # path bypasses that gate, so wait a bounded time here for the graded + # summary to arrive before exporting (otherwise a cancelled accuracy run + # can export with accuracy.* rows missing). + if ( + should_call_stop + and self._should_wait_for_accuracy + and self._accuracy_results is None + ): + await self._await_accuracy_results_for_cancel() + # Only call stop() if we were the first to trigger shutdown if should_call_stop: self.debug("Stopping system controller after profiling cancelled") await asyncio.shield(self.stop()) + async def _await_accuracy_results_for_cancel(self) -> None: + """Bounded poll for the accuracy summary on the cancel (Ctrl+C) path. + + Polls ``_accuracy_results`` up to ``Environment.ACCURACY.CANCEL_RESULT_WAIT_SEC`` + so a cancelled accuracy run still exports its graded results when the + RecordsManager's pub/sub message arrives in time; returns early as soon as + it lands, or after the bound elapses. + """ + timeout = Environment.ACCURACY.CANCEL_RESULT_WAIT_SEC + interval = 0.05 + iterations = int(timeout / interval) + for _ in range(iterations): + if self._accuracy_results is not None: + self.debug("Accuracy results arrived during cancel wait") + return + await asyncio.sleep(interval) + if self._accuracy_results is None: + self.warning( + "Accuracy results did not arrive within " + f"{timeout}s of cancellation; export may omit accuracy metrics" + ) + @on_stop async def _stop_system_controller(self) -> None: """Stop the system controller and all running services.""" diff --git a/src/aiperf/gpu_telemetry/protocols.py b/src/aiperf/gpu_telemetry/protocols.py index b10c07ebe8..0a1001c213 100644 --- a/src/aiperf/gpu_telemetry/protocols.py +++ b/src/aiperf/gpu_telemetry/protocols.py @@ -9,8 +9,8 @@ from aiperf.common.models import ErrorDetails, TelemetryRecord if TYPE_CHECKING: + from aiperf.common.accumulator_protocols import ExportContext from aiperf.common.models import ( - ErrorDetailsCount, MetricResult, TelemetryExportData, ) @@ -70,13 +70,8 @@ async def process_record(self, record: TelemetryRecord) -> None: async def summarize(self) -> list[MetricResult]: ... - def export_results( - self, - start_ns: int | None = None, - end_ns: int | None = None, - error_summary: list[ErrorDetailsCount] | None = None, - ) -> TelemetryExportData | None: - """Export accumulated telemetry data.""" + async def export_results(self, ctx: ExportContext) -> TelemetryExportData | None: + """Export accumulated telemetry data scoped to ``ctx``.""" ... def start_realtime_telemetry(self) -> None: diff --git a/src/aiperf/metrics/energy_efficiency_analyzer.py b/src/aiperf/metrics/energy_efficiency_analyzer.py index c60a14a38b..02cfc541fb 100644 --- a/src/aiperf/metrics/energy_efficiency_analyzer.py +++ b/src/aiperf/metrics/energy_efficiency_analyzer.py @@ -127,7 +127,11 @@ def metric(tag: str) -> float | None: if source is EnergySource.UNAVAILABLE or total_energy_j <= 0: return out - out.append(_result(AverageGpuPowerMetric, avg_power_w)) + # A degenerate/empty profiling window yields duration_s == 0, so the DCGM + # branch cannot derive an average power; emit total energy but skip the + # misleading 0 W average (and _per_watt_metrics likewise returns []). + if avg_power_w > 0: + out.append(_result(AverageGpuPowerMetric, avg_power_w)) out.append(_result(TotalGpuEnergyMetric, total_energy_j)) out += self._energy_ratio_metrics(total_energy_j, metric, self._concurrency()) out += self._per_watt_metrics(avg_power_w, metric) diff --git a/src/aiperf/plugin/schema/plugins.schema.json b/src/aiperf/plugin/schema/plugins.schema.json index 06ca1b2df7..f73144a25e 100644 --- a/src/aiperf/plugin/schema/plugins.schema.json +++ b/src/aiperf/plugin/schema/plugins.schema.json @@ -1082,7 +1082,7 @@ "description": "Metadata schema for record producer plugins.\n\nProducers parse a record and emit one typed result on a declared record-type\nchannel. RecordProcessorService groups producer outputs by this declared\n``record_type`` (rather than runtime type-sniffing) and routes each group to\nits dedicated downstream message.\n\nReferenced by: categories.yaml record_processor.metadata_class\nUsed in: plugins.yaml record_processor entries", "properties": { "record_type": { - "description": "The record_type channel this producer emits onto (e.g. 'metric_records', 'accuracy').", + "description": "The record_type channel this producer emits onto. Values: 'metric_records', 'accuracy', 'gpu_telemetry', 'server_metrics', 'network_latency', 'credit_phase_stats'.", "title": "Record Type", "type": "string" } @@ -1255,7 +1255,7 @@ "description": "Metadata schema for analyzer plugins.\n\nAnalyzers run at summarize time and join across accumulators. They store no\nrecords; instead they declare their dependencies by KIND:\n\n- ``required_accumulators``: needs the LIVE accumulator instance (via\n ``SummaryContext.get_accumulator``) to run a query not present in the\n summary \u2014 e.g. energy efficiency calls ``GPUTelemetryAccumulator``'s\n windowed ``total_energy_joules`` / ``total_power_watts``.\n- ``required_summaries``: needs only the already-computed summary output\n (via ``SummaryContext.get_output``) \u2014 e.g. energy efficiency reads token\n and duration totals off the metrics accumulator's summary.\n\nRecordsManager runs an analyzer only when every required accumulator is\nloaded AND every required summary was produced; otherwise it is skipped\n(e.g. energy efficiency is skipped when GPU telemetry is disabled).\n\nReferenced by: categories.yaml analyzer.metadata_class\nUsed in: plugins.yaml analyzer entries", "properties": { "required_accumulators": { - "description": "AccumulatorType names whose LIVE instance this analyzer queries via SummaryContext.get_accumulator(). The analyzer is skipped unless all are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics'.", + "description": "AccumulatorType names whose LIVE instance this analyzer queries via SummaryContext.get_accumulator(). The analyzer is skipped unless all are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency'.", "items": { "type": "string" }, @@ -1263,7 +1263,7 @@ "type": "array" }, "required_summaries": { - "description": "AccumulatorType names whose SUMMARY output this analyzer reads via SummaryContext.get_output(). The analyzer is skipped unless all were produced. Values: 'metric_records', 'gpu_telemetry', 'server_metrics'.", + "description": "AccumulatorType names whose SUMMARY output this analyzer reads via SummaryContext.get_output(). The analyzer is skipped unless all were produced. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency'.", "items": { "type": "string" }, diff --git a/src/aiperf/plugin/schema/schemas.py b/src/aiperf/plugin/schema/schemas.py index 6d38ae21a6..0c0770c06a 100644 --- a/src/aiperf/plugin/schema/schemas.py +++ b/src/aiperf/plugin/schema/schemas.py @@ -525,7 +525,9 @@ class RecordProducerMetadata(BaseModel): """ record_type: str = Field( - description="The record_type channel this producer emits onto (e.g. 'metric_records', 'accuracy').", + description="The record_type channel this producer emits onto. Values: " + "'metric_records', 'accuracy', 'gpu_telemetry', 'server_metrics', " + "'network_latency', 'credit_phase_stats'.", ) @@ -556,7 +558,8 @@ class AnalyzerMetadata(BaseModel): description=( "AccumulatorType names whose LIVE instance this analyzer queries via " "SummaryContext.get_accumulator(). The analyzer is skipped unless all " - "are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics'." + "are loaded. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', " + "'accuracy', 'network_latency'." ), ) @@ -565,7 +568,8 @@ class AnalyzerMetadata(BaseModel): description=( "AccumulatorType names whose SUMMARY output this analyzer reads via " "SummaryContext.get_output(). The analyzer is skipped unless all were " - "produced. Values: 'metric_records', 'gpu_telemetry', 'server_metrics'." + "produced. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', " + "'accuracy', 'network_latency'." ), ) diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index c8b6d569e9..d39570550a 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -414,7 +414,6 @@ def __init__( self._telemetry_state = ErrorTrackingState() self._server_metrics_state = ErrorTrackingState() - self._metric_state = ErrorTrackingState() self._gpu_telemetry_accumulator: GPUTelemetryAccumulatorProtocol | None = None self._server_metrics_accumulator: ServerMetricsAccumulatorProtocol | None = None @@ -439,6 +438,7 @@ def __init__( # each carrying its live-instance and summary dependencies. self._analyzers: list[LoadedAnalyzer] = load_analyzers(self) self._routing_table = self._build_routing_table() + self._warned_unrouted_record_types: set[str] = set() self._log_routing_table() self._metric_record_accumulators = [ @@ -488,7 +488,16 @@ async def _dispatch_record(self, record: Any) -> list[BaseException]: handlers = self._routing_table.get(record_type, []) if not handlers: - self.debug(lambda: f"No handlers registered for record type: {record_type}") + # Warn once per unrouted type: records silently vanish here while the + # request still counts as a success, so this must not stay debug-only. + if record_type not in self._warned_unrouted_record_types: + self._warned_unrouted_record_types.add(record_type) + self.warning( + f"No handlers registered for record type {record_type!r}; " + "records of this type are being dropped. Check that a producer's " + "record_type matches an accumulator/stream_exporter record_types " + "entry in plugins.yaml." + ) return [] results = await asyncio.gather( @@ -1471,13 +1480,14 @@ async def _publish_accuracy_results(self, phase: CreditPhase) -> None: inference results. Exactly-once contract: this method only runs when accuracy is enabled and - the phase is PROFILING, and it ALWAYS publishes exactly one - ``ProcessAccuracyResultMessage`` before returning. The SystemController - clears ``_should_wait_for_accuracy`` only on receipt of that message, so a - missing publish would hang shutdown forever. When there is no accumulator, - or ``export_results`` raises, a terminal ``results=None`` summary is - published instead; the success path publishes the real summary. Every path - publishes once and only once. + the phase is PROFILING. It attempts to publish exactly one + ``ProcessAccuracyResultMessage``; the SystemController clears + ``_should_wait_for_accuracy`` only on receipt of that message. A summary + that fails to export still publishes a terminal ``results=None`` message so + the gate is released. The publish itself is the only unrecoverable point: + if the message bus raises here the gate cannot be released from this side, + so we log it at error level (rather than swallowing) to make the cause of + any resulting shutdown stall diagnosable. """ summary: AccuracySummary | None = None if self._accuracy_accumulator is not None: @@ -1488,12 +1498,19 @@ async def _publish_accuracy_results(self, phase: CreditPhase) -> None: except Exception as e: # noqa: BLE001 - must still publish a terminal message self.exception(f"Accuracy summary export failed: {e!r}") summary = None - await self.publish( - ProcessAccuracyResultMessage( - service_id=self.service_id, - accuracy_result=ProcessAccuracyResult(results=summary), + try: + await self.publish( + ProcessAccuracyResultMessage( + service_id=self.service_id, + accuracy_result=ProcessAccuracyResult(results=summary), + ) ) - ) + except Exception as e: # noqa: BLE001 + self.error( + "Failed to publish ProcessAccuracyResultMessage; the controller's " + f"accuracy shutdown gate may not release: {e!r}" + ) + raise async def _process_server_metrics_results(self) -> ProcessServerMetricsResult: """Process server metrics results by exporting the accumulated server metrics data. diff --git a/src/aiperf/server_metrics/protocols.py b/src/aiperf/server_metrics/protocols.py index c676883b60..1e8d2477f1 100644 --- a/src/aiperf/server_metrics/protocols.py +++ b/src/aiperf/server_metrics/protocols.py @@ -6,8 +6,8 @@ from typing import TYPE_CHECKING, Protocol, runtime_checkable if TYPE_CHECKING: + from aiperf.common.accumulator_protocols import ExportContext from aiperf.common.models import ( - ErrorDetailsCount, MetricResult, ServerMetricsRecord, ServerMetricsResults, @@ -24,14 +24,6 @@ async def process_record(self, record: ServerMetricsRecord) -> None: async def summarize(self) -> list[MetricResult]: ... - async def export_results( - self, - start_ns: int, - end_ns: int, - error_summary: list[ErrorDetailsCount] | None = None, - *, - warmup_start_ns: int | None = None, - warmup_end_ns: int | None = None, - ) -> ServerMetricsResults | None: - """Export accumulated server metrics for a profiling window.""" + async def export_results(self, ctx: ExportContext) -> ServerMetricsResults | None: + """Export accumulated server metrics scoped to ``ctx``.""" ... diff --git a/tests/unit/records/test_records_manager_routing.py b/tests/unit/records/test_records_manager_routing.py index 37c3944861..c4de131f2f 100644 --- a/tests/unit/records/test_records_manager_routing.py +++ b/tests/unit/records/test_records_manager_routing.py @@ -205,8 +205,10 @@ class TestDispatchRecord: def _manager(self, routing_table: dict[str, list[Any]]) -> RecordsManager: manager = RecordsManager.__new__(RecordsManager) manager._routing_table = routing_table + manager._warned_unrouted_record_types = set() manager.error = MagicMock() manager.debug = MagicMock() + manager.warning = MagicMock() return manager @pytest.mark.asyncio @@ -223,14 +225,16 @@ async def test_dispatch_calls_all_handlers(self) -> None: exp.process_record.assert_awaited_once_with(record) @pytest.mark.asyncio - async def test_dispatch_with_no_handlers_is_noop(self) -> None: + async def test_dispatch_with_no_handlers_warns_once(self) -> None: manager = self._manager({}) record = MagicMock(record_type="metric_records") errors = await manager._dispatch_record(record) + # A second unrouted record of the same type must not re-warn. + errors += await manager._dispatch_record(record) assert errors == [] - manager.debug.assert_called_once() + manager.warning.assert_called_once() manager.error.assert_not_called() @pytest.mark.asyncio From c9d2c5a1be87ddb340963be84da798c1ba991e38 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 16 Jul 2026 00:35:57 -0700 Subject: [PATCH 31/39] fix(records/accuracy): resolve second-pass code-review findings Independent re-review of the post-fix accumulator-refactor branch surfaced a set of Low / Low-Medium issues; all actionable ones fixed here: - records_manager: _dispatch_record no longer re-raises a handler-level CancelledError. gather(return_exceptions=True) only captures a *child* cancellation (genuine task cancellation makes the gather itself raise), so re-raising it wrongly skipped the tracker update + timeout-less completion barrier in _on_records / _on_credit_phase_complete and could hang end-of-phase. It is now logged and counted like any other handler failure. (N1) - system_controller: cancel-path accuracy poll gates on _should_wait_for_accuracy (the flag the summary handler flips) instead of _accuracy_results, so a legitimately-empty run no longer burns the full CANCEL_RESULT_WAIT_SEC and logs a misleading warning. (N3) - energy_efficiency_analyzer: comment the grace-window/duration avg-power bias. (N4) - records_manager: delete dead _generate_realtime_metrics. (N5) - fix "record_type ClassVar" -> "serialized record_type field" in the _on_records docstring and the record_processor ship comment. (N6) - plugin schema: record_types description now lists accuracy, network_latency, credit_phase_stats; regenerate plugins.schema.json. (N7) - accuracy_record_processor: reword docstring to describe the intentional reasoning output/thinking split (does not mirror get_text()). (N8) - gpu_telemetry accumulator: drop docstring ref to removed compute_efficiency_metrics. (N9) Test test_cancelled_error_propagates rewritten to assert the corrected contract. Full unit suite green (14640 passed); accuracy e2e re-verified against the mock server (10 JSONL rows == summary total, warmup excluded). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- .../accuracy/accuracy_record_processor.py | 6 ++++-- src/aiperf/controller/system_controller.py | 14 +++++++------ src/aiperf/gpu_telemetry/accumulator.py | 6 +++--- .../metrics/energy_efficiency_analyzer.py | 6 ++++++ src/aiperf/plugin/schema/plugins.schema.json | 4 ++-- src/aiperf/plugin/schema/schemas.py | 3 ++- .../records/record_processor_service.py | 4 ++-- src/aiperf/records/records_manager.py | 21 +++++++------------ .../records/test_records_manager_routing.py | 14 ++++++++++--- 9 files changed, 46 insertions(+), 32 deletions(-) diff --git a/src/aiperf/accuracy/accuracy_record_processor.py b/src/aiperf/accuracy/accuracy_record_processor.py index 001d763fe1..78dca93379 100644 --- a/src/aiperf/accuracy/accuracy_record_processor.py +++ b/src/aiperf/accuracy/accuracy_record_processor.py @@ -186,8 +186,10 @@ def _extract_output_and_thinking( ``ReasoningResponseData.content``, or ``ToolCallResponseData`` content + ``tool_call_text``); ``model_thinking`` is the concatenated ``reasoning_content`` from any ``ReasoningResponseData`` chunks, or None - when the model emitted no separate reasoning channel. The output channel - mirrors what grading scores (``get_text()``) so the two never diverge. + when the model emitted no separate reasoning channel. For reasoning models + this deliberately splits the two channels: grading scores the full + ``get_text()`` (reasoning + content), while the export keeps ``model_output`` + to the answer content and routes reasoning to ``model_thinking``. """ output_parts: list[str] = [] thinking_parts: list[str] = [] diff --git a/src/aiperf/controller/system_controller.py b/src/aiperf/controller/system_controller.py index ee3f0f721b..510ba5c772 100644 --- a/src/aiperf/controller/system_controller.py +++ b/src/aiperf/controller/system_controller.py @@ -1001,20 +1001,22 @@ async def _cancel_profiling(self) -> None: async def _await_accuracy_results_for_cancel(self) -> None: """Bounded poll for the accuracy summary on the cancel (Ctrl+C) path. - Polls ``_accuracy_results`` up to ``Environment.ACCURACY.CANCEL_RESULT_WAIT_SEC`` - so a cancelled accuracy run still exports its graded results when the - RecordsManager's pub/sub message arrives in time; returns early as soon as - it lands, or after the bound elapses. + Waits up to ``Environment.ACCURACY.CANCEL_RESULT_WAIT_SEC`` for the + RecordsManager's pub/sub summary message to land, returning as soon as it + does. Gates on ``_should_wait_for_accuracy`` (flipped false when the + message arrives) rather than ``_accuracy_results``, since a legitimately + empty run publishes ``results=None`` -- polling the results field would + never observe that arrival and would burn the full timeout. """ timeout = Environment.ACCURACY.CANCEL_RESULT_WAIT_SEC interval = 0.05 iterations = int(timeout / interval) for _ in range(iterations): - if self._accuracy_results is not None: + if not self._should_wait_for_accuracy: self.debug("Accuracy results arrived during cancel wait") return await asyncio.sleep(interval) - if self._accuracy_results is None: + if self._should_wait_for_accuracy: self.warning( "Accuracy results did not arrive within " f"{timeout}s of cancellation; export may omit accuracy metrics" diff --git a/src/aiperf/gpu_telemetry/accumulator.py b/src/aiperf/gpu_telemetry/accumulator.py index 737dcf474e..b4c516678a 100644 --- a/src/aiperf/gpu_telemetry/accumulator.py +++ b/src/aiperf/gpu_telemetry/accumulator.py @@ -187,9 +187,9 @@ async def summarize( Async and runs periodically under the dashboard cadence (`RuntimeConfig.realtime_metrics_interval`). Emits one MetricResult per GPU per - signal. Contrast with `compute_efficiency_metrics` below, which is - sync, runs once per profiling phase, and aggregates across GPUs - into cross-GPU totals. + signal. Cross-GPU energy/power totals for the final summary are derived + separately via `total_power_watts` / `total_energy_joules` (consumed by + `EnergyEfficiencyAnalyzer`). Returns: List of MetricResult objects, one per GPU per metric type. diff --git a/src/aiperf/metrics/energy_efficiency_analyzer.py b/src/aiperf/metrics/energy_efficiency_analyzer.py index 02cfc541fb..e442b8ae42 100644 --- a/src/aiperf/metrics/energy_efficiency_analyzer.py +++ b/src/aiperf/metrics/energy_efficiency_analyzer.py @@ -110,6 +110,12 @@ def metric(tag: str) -> float | None: start_ns = ctx.start_ns or None end_ns = ctx.end_ns or None + # DCGM average power = energy / duration. The energy numerator integrates + # over [start, end + FINAL_SCRAPE_GRACE_NS) to capture the trailing scrape, + # while this denominator is the exact profiling window, so avg power is + # biased slightly high (~grace/duration; negligible past a few seconds, up + # to a few percent on very short runs). Exact alignment would require the + # accumulator to expose the actual first/last scrape timestamps. duration_s = ( (ctx.end_ns - ctx.start_ns) / NANOS_PER_SECOND if ctx.end_ns > ctx.start_ns diff --git a/src/aiperf/plugin/schema/plugins.schema.json b/src/aiperf/plugin/schema/plugins.schema.json index f73144a25e..213a88790d 100644 --- a/src/aiperf/plugin/schema/plugins.schema.json +++ b/src/aiperf/plugin/schema/plugins.schema.json @@ -1165,7 +1165,7 @@ "description": "Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries", "properties": { "record_types": { - "description": "Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics'.", + "description": "Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.", "items": { "type": "string" }, @@ -1210,7 +1210,7 @@ "description": "Metadata schema for record routing in accumulator and stream exporter plugins.\n\nDefines which record types an accumulator or stream exporter accepts. Used by\nRecordsManager to build a routing table: incoming records are dispatched to all\naccumulators and stream exporters whose record_types include the matching type.\nThe role (accumulator vs stream_exporter) is determined by the plugin category.\n\nReferenced by: categories.yaml accumulator.metadata_class, stream_exporter.metadata_class\nUsed in: plugins.yaml accumulator and stream_exporter entries", "properties": { "record_types": { - "description": "Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics'.", + "description": "Record type identifiers this accumulator or stream exporter accepts for routing. RecordsManager dispatches incoming records to all accumulators and stream exporters whose record_types include the matching type. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency', 'credit_phase_stats'.", "items": { "type": "string" }, diff --git a/src/aiperf/plugin/schema/schemas.py b/src/aiperf/plugin/schema/schemas.py index 0c0770c06a..bbe54b39bc 100644 --- a/src/aiperf/plugin/schema/schemas.py +++ b/src/aiperf/plugin/schema/schemas.py @@ -507,7 +507,8 @@ class RecordRoutingMetadata(BaseModel): "Record type identifiers this accumulator or stream exporter accepts for routing. " "RecordsManager dispatches incoming records to all accumulators and stream exporters " "whose record_types include the matching type. " - "Values: 'metric_records', 'gpu_telemetry', 'server_metrics'." + "Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', " + "'network_latency', 'credit_phase_stats'." ), ) diff --git a/src/aiperf/records/record_processor_service.py b/src/aiperf/records/record_processor_service.py index ad1fc4bfa7..436319b6ea 100644 --- a/src/aiperf/records/record_processor_service.py +++ b/src/aiperf/records/record_processor_service.py @@ -350,8 +350,8 @@ async def _process_and_forward_record( # Ship generically: ONE RecordsMessage per inference record carries the # request envelope (metadata + request-level error) plus every produced # typed record flattened into one list. Each record self-identifies via - # its own record_type ClassVar, so no per-type message class or builder - # map is needed. Always pushed (even when no producer emitted a record) + # its own serialized record_type field, so no per-type message class or + # builder map is needed. Always pushed (even when no producer emitted a record) # to keep the RecordsManager completion barrier in lockstep with the # credit. The metric producer already put trace_data inside its # MetricRecordsData, so trace_data is not carried on the envelope. diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index d39570550a..5f363d9897 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -506,8 +506,12 @@ async def _dispatch_record(self, record: Any) -> list[BaseException]: ) errors: list[BaseException] = [] for handler, result in zip(handlers, results, strict=True): - if isinstance(result, asyncio.CancelledError): - raise result + # A handler-level CancelledError (captured by return_exceptions) means + # one handler's coroutine was cancelled, NOT this task -- genuine task + # cancellation makes the gather itself raise and never reaches here. We + # must count it like any other handler failure rather than re-raising, + # or the caller skips the tracker update + (timeout-less) completion + # barrier and the phase never converges. if isinstance(result, BaseException): self.error( f"Handler {handler.__class__.__name__} failed for " @@ -534,8 +538,8 @@ async def _on_records(self, message: RecordsMessage) -> None: """Handle a per-request records envelope generically. One ``RecordsMessage`` == one inference request. Each contained record - self-identifies via its ``record_type`` ClassVar and is dispatched to its - registered handlers; the per-request lockstep keys off the message + self-identifies via its serialized ``record_type`` field and is dispatched + to its registered handlers; the per-request lockstep keys off the message envelope (``message.metadata`` / ``message.error``), never off sniffing a record type. """ @@ -1023,15 +1027,6 @@ async def _report_realtime_metrics( for line in rendered.splitlines(): self.info(line) - async def _generate_realtime_metrics(self) -> list[MetricResult]: - """Generate the real-time metrics for the profile run. - - Sources from the metric_records accumulators (the active summary - engine). Tolerates both AccumulatorMetricsSummary (.results dict) and - plain list[MetricResult] shapes via the shared helper. - """ - return await generate_realtime_metrics(self._metric_record_accumulators) - async def _run_analyzers(self, ctx: SummaryContext) -> list[MetricResult]: """Run summarize-time analyzer plugins that join across accumulators. diff --git a/tests/unit/records/test_records_manager_routing.py b/tests/unit/records/test_records_manager_routing.py index c4de131f2f..59ff05b90e 100644 --- a/tests/unit/records/test_records_manager_routing.py +++ b/tests/unit/records/test_records_manager_routing.py @@ -278,13 +278,21 @@ async def test_dispatch_multiple_handler_exceptions(self) -> None: assert manager.error.call_count == 2 @pytest.mark.asyncio - async def test_cancelled_error_propagates(self) -> None: + async def test_handler_cancelled_error_is_counted_not_reraised(self) -> None: + # A handler-level CancelledError (captured by gather's return_exceptions) + # means one handler's coroutine was cancelled, not this task. It must be + # returned as an error -- not re-raised -- so the caller still advances the + # tracker update and the (timeout-less) completion barrier. Genuine task + # cancellation makes the gather itself raise and never reaches this code. acc = StubAccumulator() acc.process_record.side_effect = asyncio.CancelledError manager = self._manager({"metric_records": [acc]}) - with pytest.raises(asyncio.CancelledError): - await manager._dispatch_record(MagicMock(record_type="metric_records")) + errors = await manager._dispatch_record(MagicMock(record_type="metric_records")) + + assert len(errors) == 1 + assert isinstance(errors[0], asyncio.CancelledError) + manager.error.assert_called_once() # --------------------------------------------------------------------------- From a9219528a9b8de0c8f3743f3c161a7b1cee47b3b Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 16 Jul 2026 00:52:27 -0700 Subject: [PATCH 32/39] fix(records/accuracy): single-flight results processing + aligned accuracy tasks Resolves the two remaining second-pass review findings that were code bugs (the third, N2, is intentional framework behavior -- see below): - records_manager: _process_results is now single-flight per phase. The natural finalize task, the PROCESS_RECORDS command and PROFILE_CANCEL could all reach it concurrently and double-publish results + double-finalize (close/flush) every stream exporter. An asyncio.Lock serializes them and a per-phase cache returns the first result to later callers. Body moved to _process_results_impl. (N10) - accuracy_record_processor: on_dataset_configured now builds _ground_truths and _tasks from the same graded-conversation list so they stay index-aligned. The old code filtered tasks independently (dropping label-less ones), which shifted the session_num modulo and mismapped records to the wrong task whenever only some conversations carried an accuracy_task label. Label-less graded conversations now keep a None slot. (N11) - auto_routed_model: documented (comment only) that an unregistered discriminator intentionally falls back to the base class -- the command hierarchy routes generic commands with no dedicated subclass to CommandMessage/ CommandSuccessResponse, and a test asserts this. Record subclasses rely on their model being imported (registered via __init_subclass__) instead. (N2, won't-fix) Tests: new TestProcessResultsSingleFlight and test_tasks_stay_aligned_with_ground_truths_when_labels_are_sparse; existing process-results harnesses updated for the impl split. Full unit suite green (14642 passed); accuracy e2e re-verified (10 JSONL rows == summary total). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- .../accuracy/accuracy_record_processor.py | 18 +++++----- src/aiperf/common/models/auto_routed_model.py | 5 +++ src/aiperf/records/records_manager.py | 28 +++++++++++++++ .../test_accuracy_record_processor.py | 29 ++++++++++++++++ tests/unit/records/test_records_manager.py | 3 ++ .../test_records_manager_process_results.py | 34 +++++++++++++++++++ 6 files changed, 108 insertions(+), 9 deletions(-) diff --git a/src/aiperf/accuracy/accuracy_record_processor.py b/src/aiperf/accuracy/accuracy_record_processor.py index 78dca93379..570325b260 100644 --- a/src/aiperf/accuracy/accuracy_record_processor.py +++ b/src/aiperf/accuracy/accuracy_record_processor.py @@ -68,16 +68,16 @@ def on_dataset_configured(self, metadata: DatasetMetadata) -> None: Builds the ordered list of ground-truth answers from ConversationMetadata so that process_record can grade without re-loading the benchmark. """ - self._ground_truths = [ - c.accuracy_ground_truth - for c in metadata.conversations - if c.accuracy_ground_truth is not None - ] - self._tasks = [ - c.accuracy_task - for c in metadata.conversations - if c.accuracy_task is not None + # Build ground-truth and task lists from the SAME graded conversations so + # they stay index-aligned. Filtering tasks independently (dropping the + # label-less ones) would shift the modulo and mismap records to the wrong + # task whenever only some conversations carry an accuracy_task label. A + # graded conversation with no task label keeps a None entry here. + graded = [ + c for c in metadata.conversations if c.accuracy_ground_truth is not None ] + self._ground_truths = [c.accuracy_ground_truth for c in graded] + self._tasks = [c.accuracy_task for c in graded] async def process_record( self, record: ParsedResponseRecord, metadata: MetricRecordMetadata diff --git a/src/aiperf/common/models/auto_routed_model.py b/src/aiperf/common/models/auto_routed_model.py index 52c1b1a814..9193e63229 100644 --- a/src/aiperf/common/models/auto_routed_model.py +++ b/src/aiperf/common/models/auto_routed_model.py @@ -106,4 +106,9 @@ def from_json(cls, json_or_dict: str | bytes | bytearray | dict) -> Self: # Recurse for nested routing, otherwise validate return target_class.from_json(data) + # An unregistered discriminator falls back to the base class by design: + # the command hierarchy routes generic commands (no dedicated subclass) to + # the base CommandMessage/CommandSuccessResponse. Record subclasses with + # required fields rely on their model being imported so it registers via + # __init_subclass__ (see RecordData) rather than on this fallback. return cls.model_validate(data) diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 5f363d9897..a817735e08 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -441,6 +441,12 @@ def __init__( self._warned_unrouted_record_types: set[str] = set() self._log_routing_table() + # Single-flight guard for _process_results: the background finalize task, + # the PROCESS_RECORDS command, and PROFILE_CANCEL can all reach it and + # would otherwise double-publish and double-finalize stream exporters. + self._process_results_lock = asyncio.Lock() + self._processed_results: dict[CreditPhase, ProcessRecordsResult] = {} + self._metric_record_accumulators = [ accumulator for accumulator in self._accumulators.values() @@ -1320,6 +1326,28 @@ def _deliver_network_rtt_to_accumulators(self) -> None: async def _process_results( self, phase: CreditPhase, cancelled: bool + ) -> ProcessRecordsResult: + """Process the accumulated records into final benchmark results. + + Single-flight: the natural finalize task and the PROCESS_RECORDS / + PROFILE_CANCEL commands can race. The lock serializes them and the + per-phase cache makes every call after the first return the same result + instead of re-publishing and re-finalizing the stream exporters. + """ + async with self._process_results_lock: + cached = self._processed_results.get(phase) + if cached is not None: + self.debug( + lambda: f"Results for phase {phase} already processed; " + "returning cached result" + ) + return cached + result = await self._process_results_impl(phase, cancelled) + self._processed_results[phase] = result + return result + + async def _process_results_impl( + self, phase: CreditPhase, cancelled: bool ) -> ProcessRecordsResult: """Process the accumulated records into final benchmark results.""" self.debug(lambda: f"Processing records (cancelled: {cancelled})") diff --git a/tests/unit/accuracy/test_accuracy_record_processor.py b/tests/unit/accuracy/test_accuracy_record_processor.py index db6da2bf65..557b15b251 100644 --- a/tests/unit/accuracy/test_accuracy_record_processor.py +++ b/tests/unit/accuracy/test_accuracy_record_processor.py @@ -91,6 +91,35 @@ def test_skips_conversations_without_accuracy_fields(self, monkeypatch) -> None: assert processor._ground_truths == ["B"] + def test_tasks_stay_aligned_with_ground_truths_when_labels_are_sparse( + self, monkeypatch + ) -> None: + """Graded conversations missing a task label keep a None slot so the + session_num modulo maps to the correct task instead of being shifted.""" + processor = _make_processor(monkeypatch) + conversations = [ + ConversationMetadata( + conversation_id="c0", accuracy_ground_truth="A", accuracy_task="t0" + ), + ConversationMetadata( + conversation_id="c1", accuracy_ground_truth="B" + ), # graded, no task label + ConversationMetadata( + conversation_id="c2", accuracy_ground_truth="C", accuracy_task="t2" + ), + ] + metadata = DatasetMetadata( + conversations=conversations, + sampling_strategy=DatasetSamplingStrategy.SEQUENTIAL, + ) + + processor.on_dataset_configured(metadata) + + assert processor._ground_truths == ["A", "B", "C"] + # Index-aligned with ground truths; the label-less conversation is None, + # not dropped (which would have mismapped c1 -> "t2"). + assert processor._tasks == ["t0", None, "t2"] + @pytest.mark.asyncio class TestAccuracyRecordProcessorSessionBounds: diff --git a/tests/unit/records/test_records_manager.py b/tests/unit/records/test_records_manager.py index 37e1f897e3..69ae468c8f 100644 --- a/tests/unit/records/test_records_manager.py +++ b/tests/unit/records/test_records_manager.py @@ -676,6 +676,9 @@ async def test_completed_excludes_analyzer_metrics(self) -> None: manager._error_tracker = MagicMock() manager._error_tracker.get_error_summary_for_phase.return_value = [] + manager._process_results_lock = asyncio.Lock() + manager._processed_results = {} + result = await manager._process_results(CreditPhase.PROFILING, cancelled=False) assert result.results.completed == len(request_records) diff --git a/tests/unit/records/test_records_manager_process_results.py b/tests/unit/records/test_records_manager_process_results.py index f55cc7d127..249b99eafd 100644 --- a/tests/unit/records/test_records_manager_process_results.py +++ b/tests/unit/records/test_records_manager_process_results.py @@ -19,6 +19,7 @@ from __future__ import annotations +import asyncio from unittest.mock import AsyncMock, MagicMock import pytest @@ -154,8 +155,13 @@ def _make_manager_mock( mgr.service_id = "test_records_manager" mgr.publish = AsyncMock() + # Single-flight guard state read by the real _process_results wrapper. + mgr._process_results_lock = asyncio.Lock() + mgr._processed_results = {} + # Bind real methods mgr._process_results = RecordsManager._process_results.__get__(mgr) + mgr._process_results_impl = RecordsManager._process_results_impl.__get__(mgr) mgr._summarize_metric_record_accumulators = ( RecordsManager._summarize_metric_record_accumulators.__get__(mgr) ) @@ -398,6 +404,34 @@ async def test_publishes_process_all_results_message(self) -> None: assert msg is not None +class TestProcessResultsSingleFlight: + """``_process_results`` is single-flight per phase: the natural finalize task + and the PROCESS_RECORDS / PROFILE_CANCEL commands can all reach it, but only + the first does the work; later calls return the cached result without + re-publishing or re-finalizing the stream exporters.""" + + @pytest.mark.asyncio + async def test_second_call_returns_cached_and_does_not_republish(self) -> None: + acc = _make_summary_accumulator([_STUB_METRIC_RESULT]) + exp = _make_stub_stream_exporter() + mgr = _make_manager_mock( + accumulators={AccumulatorType.METRIC_RESULTS: acc}, + stream_exporters={StreamExporterType.RECORD_EXPORT: exp}, + ) + + first = await mgr._process_results(phase=CreditPhase.PROFILING, cancelled=False) + publish_count = mgr.publish.await_count + + second = await mgr._process_results( + phase=CreditPhase.PROFILING, cancelled=False + ) + + assert second is first + # No additional publishes and the exporter is finalized exactly once. + assert mgr.publish.await_count == publish_count + exp.finalize.assert_awaited_once() + + class TestProcessResultsServerMetricsFailure: """A server-metrics processing failure must not abort the fan-in publish.""" From 9c4062a0ba9afa22cc9ab7a187f2f14ad60a694a Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 16 Jul 2026 07:57:38 -0700 Subject: [PATCH 33/39] fix(records): address accumulator-refactor code-review findings Apply the code-review findings on the accumulator/records refactor: - records_manager: skip best-effort handler (OTel/MLflow) failures in _dispatch_record so a downed collector can't inflate the phase error summary (the is_best_effort marker was dead after the refactor). Add a warn_if_unrouted flag and pass False for the credit_phase_stats dispatches -- those stats are applied via update_phase_info before dispatch and only routed to the default-absent OTel streamer, so every non-OTel run was falsely warning 'records of this type are being dropped'. - records_manager_processing: delete the dead, stale build_process_records_result duplicate and the imports it alone kept alive. - record_processor_service: snapshot the wire payload before observers run so a misbehaving observer can't reshape what RecordsManager ingests. - auto_routed_model/record_models: add a strict_routing flag; RecordData now raises on an unregistered record_type instead of degrading to a shapeless base. - gpu_telemetry/energy_efficiency_analyzer: derive duration from the telemetry scrape span (min/max) on an unbounded (full-range) window so avg power and per-watt metrics stay consistent with total energy; document the display-unit coupling on _result. - schema/records_manager: correct the required_summaries note (only the metric_results summary is registered) and the accuracy-publish docstring (bounded wait is cancel-path only; drop the misleading re-raise). - tests: rename a codespell-flagged local (datas -> data_list); add regression tests for the best-effort dispatch skip, control-plane no-warn, and RecordData strict routing. All unit tests pass; verified end-to-end against the mock server (accuracy channel, warmup phase scoping, and the suppressed control-plane warning). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/common/models/auto_routed_model.py | 20 ++++++-- src/aiperf/common/models/record_models.py | 3 ++ src/aiperf/gpu_telemetry/accumulator.py | 15 ++++++ .../metrics/energy_efficiency_analyzer.py | 18 ++++++- src/aiperf/plugin/schema/plugins.schema.json | 2 +- src/aiperf/plugin/schema/schemas.py | 8 +++- .../record_observer_context.py | 4 +- .../records/record_processor_service.py | 10 +++- src/aiperf/records/records_manager.py | 43 ++++++++++++----- .../records/records_manager_processing.py | 48 +------------------ .../test_accuracy_record_processor.py | 4 +- .../unit/common/models/test_record_models.py | 40 ++++++++++++++++ tests/unit/records/test_records_manager.py | 10 ++-- .../records/test_records_manager_routing.py | 32 +++++++++++++ 14 files changed, 183 insertions(+), 74 deletions(-) diff --git a/src/aiperf/common/models/auto_routed_model.py b/src/aiperf/common/models/auto_routed_model.py index 9193e63229..57b2e7a4f0 100644 --- a/src/aiperf/common/models/auto_routed_model.py +++ b/src/aiperf/common/models/auto_routed_model.py @@ -50,6 +50,11 @@ class AutoRoutedModel(BaseModel): discriminator_field: ClassVar[str | None] = None _model_lookup_table: ClassVar[dict[Any, type[Self]]] = {} + # When True, an unregistered discriminator value raises instead of falling + # back to ``cls.model_validate`` on the base. Set on hierarchies whose base has + # no meaningful standalone shape (e.g. RecordData), so a record whose type + # isn't registered fails loudly rather than silently dropping its typed fields. + strict_routing: ClassVar[bool] = False def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) @@ -106,9 +111,18 @@ def from_json(cls, json_or_dict: str | bytes | bytearray | dict) -> Self: # Recurse for nested routing, otherwise validate return target_class.from_json(data) + if cls.strict_routing: + # No standalone base shape: an unregistered type almost always + # means the producing module wasn't imported (registration happens + # via __init_subclass__), so fail loudly instead of degrading. + raise ValueError( + f"Unknown {cls_discriminator} {discriminator_value!r}: no " + f"{cls.__name__} subclass is registered for it. Ensure the " + "producing module is imported so it registers via " + "__init_subclass__." + ) + # An unregistered discriminator falls back to the base class by design: # the command hierarchy routes generic commands (no dedicated subclass) to - # the base CommandMessage/CommandSuccessResponse. Record subclasses with - # required fields rely on their model being imported so it registers via - # __init_subclass__ (see RecordData) rather than on this fallback. + # the base CommandMessage/CommandSuccessResponse. return cls.model_validate(data) diff --git a/src/aiperf/common/models/record_models.py b/src/aiperf/common/models/record_models.py index 7470a57c2f..bb6f958235 100644 --- a/src/aiperf/common/models/record_models.py +++ b/src/aiperf/common/models/record_models.py @@ -138,6 +138,9 @@ class RecordData(AIPerfBaseModel): """ discriminator_field: ClassVar[str] = "record_type" + # The base RecordData has no standalone shape (typed fields live on subclasses), + # so an unregistered record_type must raise rather than silently degrade. + strict_routing: ClassVar[bool] = True record_type: str = Field( description="Discriminator: the record_type channel this record routes on.", diff --git a/src/aiperf/gpu_telemetry/accumulator.py b/src/aiperf/gpu_telemetry/accumulator.py index b4c516678a..05af2f7883 100644 --- a/src/aiperf/gpu_telemetry/accumulator.py +++ b/src/aiperf/gpu_telemetry/accumulator.py @@ -104,6 +104,21 @@ def query_time_range(self, start_ns: int, end_ns: int) -> NDArray[np.bool_]: ts = self._timestamps_ns.data return (ts >= start_ns) & (ts < end_ns) + def scrape_span_ns(self) -> tuple[int, int] | None: + """Return ``(first_ns, last_ns)`` of collected scrapes, or None if empty. + + Analyzer support: lets a caller recover the actual observed telemetry + window when it holds an unbounded (full-range) summary window and needs a + real duration (e.g. to derive average power without a bounded phase). + """ + if len(self._timestamps_ns) == 0: + return None + ts = self._timestamps_ns.data + # min/max rather than [0]/[-1]: scrapes are appended in collection order, + # which is normally monotonic but not guaranteed (retries/out-of-order + # arrival), so don't assume the ends are the extremes. + return int(ts.min()), int(ts.max()) + def start_realtime_telemetry(self) -> None: """Start the realtime telemetry background task. diff --git a/src/aiperf/metrics/energy_efficiency_analyzer.py b/src/aiperf/metrics/energy_efficiency_analyzer.py index e442b8ae42..4419b0714c 100644 --- a/src/aiperf/metrics/energy_efficiency_analyzer.py +++ b/src/aiperf/metrics/energy_efficiency_analyzer.py @@ -57,7 +57,14 @@ class EnergySource(str, enum.Enum): def _result(metric_cls: type, value: float) -> MetricResult: - """Build the injected MetricResult for an energy metric class.""" + """Build the injected MetricResult for an energy metric class. + + ``value`` is emitted in ``metric_cls.unit`` verbatim: analyzer output bypasses + ``MetricsAccumulator._convert_display_units``, so any ``display_unit`` on one of + these classes would be silently ignored. Keep ``display_unit == unit`` (i.e. + don't declare a ``display_unit``) on the power_efficiency_metrics classes, or + route this through unit conversion first. + """ return MetricResult( tag=metric_cls.tag, header=metric_cls.header, @@ -121,6 +128,15 @@ def metric(tag: str) -> float | None: if ctx.end_ns > ctx.start_ns else 0.0 ) + # An unbounded (full-range) window leaves start==end==0 -> duration_s==0, + # which would drop average power and every per-watt metric while total + # energy still emits (a half-populated family). Recover a real duration + # from the observed telemetry scrape span so the whole family stays + # consistent. A genuinely empty accumulator still yields 0.0. + if duration_s <= 0.0: + span = gpu.scrape_span_ns() + if span is not None and span[1] > span[0]: + duration_s = (span[1] - span[0]) / NANOS_PER_SECOND energy = gpu.total_energy_joules(start_ns, end_ns) power = gpu.total_power_watts(start_ns, end_ns) total_energy_j, avg_power_w, source = self._resolve_energy( diff --git a/src/aiperf/plugin/schema/plugins.schema.json b/src/aiperf/plugin/schema/plugins.schema.json index 213a88790d..46c237a2a8 100644 --- a/src/aiperf/plugin/schema/plugins.schema.json +++ b/src/aiperf/plugin/schema/plugins.schema.json @@ -1263,7 +1263,7 @@ "type": "array" }, "required_summaries": { - "description": "AccumulatorType names whose SUMMARY output this analyzer reads via SummaryContext.get_output(). The analyzer is skipped unless all were produced. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', 'accuracy', 'network_latency'.", + "description": "AccumulatorType names whose SUMMARY output this analyzer reads via SummaryContext.get_output(). The analyzer is skipped unless all were produced. NOTE: only the 'metric_results' summary is currently registered into SummaryContext.accumulator_outputs; side-channel accumulators (gpu_telemetry, server_metrics, accuracy, network_latency) summarize separately and are NOT available here -- depend on their LIVE instance via required_accumulators instead. Declaring one of those in required_summaries silently skips the analyzer every run.", "items": { "type": "string" }, diff --git a/src/aiperf/plugin/schema/schemas.py b/src/aiperf/plugin/schema/schemas.py index bbe54b39bc..a7518ad769 100644 --- a/src/aiperf/plugin/schema/schemas.py +++ b/src/aiperf/plugin/schema/schemas.py @@ -569,8 +569,12 @@ class AnalyzerMetadata(BaseModel): description=( "AccumulatorType names whose SUMMARY output this analyzer reads via " "SummaryContext.get_output(). The analyzer is skipped unless all were " - "produced. Values: 'metric_records', 'gpu_telemetry', 'server_metrics', " - "'accuracy', 'network_latency'." + "produced. NOTE: only the 'metric_results' summary is currently " + "registered into SummaryContext.accumulator_outputs; side-channel " + "accumulators (gpu_telemetry, server_metrics, accuracy, network_latency) " + "summarize separately and are NOT available here -- depend on their LIVE " + "instance via required_accumulators instead. Declaring one of those in " + "required_summaries silently skips the analyzer every run." ), ) diff --git a/src/aiperf/post_processors/record_observer_context.py b/src/aiperf/post_processors/record_observer_context.py index d91e3c5f94..bc5769210f 100644 --- a/src/aiperf/post_processors/record_observer_context.py +++ b/src/aiperf/post_processors/record_observer_context.py @@ -30,7 +30,9 @@ class RecordObserverContext: """The metric record metadata for this record.""" produced: dict[str, list[Any]] - """Producer outputs keyed by declared record_type. + """Producer outputs keyed by declared record_type. Read-only by contract: + the wire payload is snapshotted before observers run, so mutating this cannot + change what RecordsManager ingests. e.g. ``{"metric_records": [MetricRecordDict], "accuracy": [AccuracyRecordsData]}``. """ diff --git a/src/aiperf/records/record_processor_service.py b/src/aiperf/records/record_processor_service.py index 436319b6ea..56644e57fd 100644 --- a/src/aiperf/records/record_processor_service.py +++ b/src/aiperf/records/record_processor_service.py @@ -327,6 +327,11 @@ async def _process_and_forward_record( continue by_type.setdefault(record_type, []).append(result) + # Snapshot the wire payload BEFORE observers run: ``produced`` is read-only + # by contract, but a misbehaving observer that mutated ``by_type`` must not + # be able to change what RecordsManager ingests. + all_records = [record for records in by_type.values() for record in records] + # Stage 2 - observers: view the produced results + the record and act. # Must run BEFORE _free_record_data so they can read the full parsed # record via ctx.record. @@ -355,7 +360,10 @@ async def _process_and_forward_record( # to keep the RecordsManager completion barrier in lockstep with the # credit. The metric producer already put trace_data inside its # MetricRecordsData, so trace_data is not carried on the envelope. - all_records = [record for records in by_type.values() for record in records] + # The push is atomic (whole message serialized, then one NOBLOCK frame + # send): a failure delivers nothing, so it propagates to the outer handler + # which forwards exactly one error record -- no partial send, no + # double-count, and the completion barrier stays in lockstep. await self.records_push_client.push( RecordsMessage( service_id=self.service_id, diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index a817735e08..7138a7bd71 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -484,8 +484,18 @@ def _build_routing_table(self) -> dict[str, list[Any]]: table.setdefault(record_type, []).append(handler) return table - async def _dispatch_record(self, record: Any) -> list[BaseException]: - """Dispatch one typed record to all handlers registered for its record_type.""" + async def _dispatch_record( + self, record: Any, *, warn_if_unrouted: bool = True + ) -> list[BaseException]: + """Dispatch one typed record to all handlers registered for its record_type. + + ``warn_if_unrouted`` gates the "no handlers" warning: leave it True for + data-plane records (which would be silently lost), but set it False for + control-plane records that are already consumed elsewhere and only + OPTIONALLY streamed (e.g. ``credit_phase_stats``, whose stats are applied + via ``update_phase_info`` before dispatch and only routed to the + default-absent OTel streamer) -- otherwise every non-OTel run warns falsely. + """ record_type = getattr(record, "record_type", None) if record_type is None: error = TypeError(f"Record {type(record).__name__} has no record_type") @@ -496,7 +506,10 @@ async def _dispatch_record(self, record: Any) -> list[BaseException]: if not handlers: # Warn once per unrouted type: records silently vanish here while the # request still counts as a success, so this must not stay debug-only. - if record_type not in self._warned_unrouted_record_types: + if ( + warn_if_unrouted + and record_type not in self._warned_unrouted_record_types + ): self._warned_unrouted_record_types.add(record_type) self.warning( f"No handlers registered for record type {record_type!r}; " @@ -523,6 +536,12 @@ async def _dispatch_record(self, record: Any) -> list[BaseException]: f"Handler {handler.__class__.__name__} failed for " f"{record_type}: {result!r}" ) + # Best-effort handlers (streaming telemetry like OTel/MLflow) must + # never pollute the benchmark's phase error count -- a downed + # collector is not an inference failure. Log and drop, honoring the + # handler's ``is_best_effort`` marker. + if getattr(handler, "is_best_effort", False): + continue errors.append(result) return errors @@ -705,7 +724,7 @@ async def _on_credit_phase_start( ) -> None: """Handle a credit phase start message in order to track the total number of expected requests.""" self._records_tracker.update_phase_info(phase_start_msg.stats) - await self._dispatch_record(phase_start_msg.stats) + await self._dispatch_record(phase_start_msg.stats, warn_if_unrouted=False) self.info(f"Credit phase start: {phase_start_msg.config.phase}") @on_message(MessageType.CREDIT_PHASE_PROGRESS) @@ -714,7 +733,7 @@ async def _on_credit_phase_progress( ) -> None: """Handle a credit phase progress message to track and stream live timing snapshots.""" self._records_tracker.update_phase_info(message.stats) - await self._dispatch_record(message.stats) + await self._dispatch_record(message.stats, warn_if_unrouted=False) @on_message(MessageType.CREDIT_PHASE_SENDING_COMPLETE) async def _on_credit_phase_sending_complete( @@ -726,7 +745,7 @@ async def _on_credit_phase_sending_complete( f"Sent {message.stats.final_requests_sent:,} requests. Waiting for all to complete..." ) self._records_tracker.update_phase_info(message.stats) - await self._dispatch_record(message.stats) + await self._dispatch_record(message.stats, warn_if_unrouted=False) @on_message(MessageType.CREDIT_PHASE_COMPLETE) async def _on_credit_phase_complete( @@ -734,7 +753,7 @@ async def _on_credit_phase_complete( ) -> None: """Handle a credit phase complete message in order to track the end time, and check if all records have been received.""" self._records_tracker.update_phase_info(message.stats) - await self._dispatch_record(message.stats) + await self._dispatch_record(message.stats, warn_if_unrouted=False) self._complete_credit_phases.add(message.stats.phase) # Capture per-phase BranchStats for any phase that publishes them. if message.branch_stats is not None: @@ -1508,9 +1527,12 @@ async def _publish_accuracy_results(self, phase: CreditPhase) -> None: ``_should_wait_for_accuracy`` only on receipt of that message. A summary that fails to export still publishes a terminal ``results=None`` message so the gate is released. The publish itself is the only unrecoverable point: - if the message bus raises here the gate cannot be released from this side, - so we log it at error level (rather than swallowing) to make the cause of - any resulting shutdown stall diagnosable. + if the message bus raises here the gate cannot be released from this side. + We log it at error level and return rather than re-raising -- propagating + would skip the subsequent ``ProcessAllResultsMessage`` publish, and the + caller already logs it. (Note: only the CANCEL path has a bounded wait on + this gate; on normal completion an unreleased gate stalls shutdown, so this + error log is the primary diagnostic for that case.) """ summary: AccuracySummary | None = None if self._accuracy_accumulator is not None: @@ -1533,7 +1555,6 @@ async def _publish_accuracy_results(self, phase: CreditPhase) -> None: "Failed to publish ProcessAccuracyResultMessage; the controller's " f"accuracy shutdown gate may not release: {e!r}" ) - raise async def _process_server_metrics_results(self) -> ProcessServerMetricsResult: """Process server metrics results by exporting the accumulated server metrics data. diff --git a/src/aiperf/records/records_manager_processing.py b/src/aiperf/records/records_manager_processing.py index f1ebe09171..73f44a774f 100644 --- a/src/aiperf/records/records_manager_processing.py +++ b/src/aiperf/records/records_manager_processing.py @@ -12,7 +12,6 @@ from __future__ import annotations import asyncio -import time from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Protocol @@ -20,13 +19,7 @@ from aiperf.common.enums import CreditPhase, MetricConsoleGroup, MetricFlags from aiperf.common.exceptions import PluginDisabled, PostProcessorDisabled from aiperf.common.logging import AIPerfLogger -from aiperf.common.models import ( - ErrorDetails, - MetricResult, - ProcessRecordsResult, - ProfileResults, - TimesliceResult, -) +from aiperf.common.models import MetricResult from aiperf.plugin import plugins from aiperf.plugin.enums import ( AccumulatorType, @@ -40,10 +33,7 @@ AnalyzerProtocol, StreamExporterProtocol, ) - from aiperf.common.models.branch_stats import BranchStats from aiperf.config.resolution.plan import BenchmarkRun - from aiperf.records.error_tracker import ErrorTracker - from aiperf.records.records_tracker import RecordsTracker _logger = AIPerfLogger(__name__) @@ -275,39 +265,3 @@ def filter_display_metrics(raw_metrics: list[MetricResult]) -> list[MetricResult pass display_metrics.append(m) return display_metrics - - -def build_process_records_result( - *, - records_results: list[MetricResult], - warmup_records_results: list[MetricResult] | None = None, - timeslices: list[TimesliceResult], - error_results: list[ErrorDetails], - tracker: RecordsTracker, - error_tracker: ErrorTracker, - cancelled: bool, - branch_stats: BranchStats | None = None, -) -> ProcessRecordsResult: - """Assemble the final ``ProcessRecordsResult`` from accumulator output. - - Single-phase ``CreditPhase.PROFILING`` model — ``RecordsTracker`` does - not expose a multi-phase ``get_results_phases`` / - ``get_results_time_window`` API. - """ - phase_stats = tracker.create_stats_for_phase(CreditPhase.PROFILING) - return ProcessRecordsResult( - results=ProfileResults( - records=records_results, - warmup_records=warmup_records_results or None, - timeslices=timeslices or None, - completed=len(records_results), - start_ns=phase_stats.start_ns or time.time_ns(), - end_ns=phase_stats.requests_end_ns or time.time_ns(), - error_summary=error_tracker.get_error_summary_for_phase( - CreditPhase.PROFILING - ), - was_cancelled=cancelled, - branch_stats=branch_stats, - ), - errors=error_results, - ) diff --git a/tests/unit/accuracy/test_accuracy_record_processor.py b/tests/unit/accuracy/test_accuracy_record_processor.py index 557b15b251..c58c16706f 100644 --- a/tests/unit/accuracy/test_accuracy_record_processor.py +++ b/tests/unit/accuracy/test_accuracy_record_processor.py @@ -292,7 +292,7 @@ class TestExtractOutputAndThinking: """`_extract_output_and_thinking` splits answer content from reasoning.""" @staticmethod - def _record(datas: list) -> "object": + def _record(data_list: list) -> "object": from aiperf.common.models.record_models import ( ParsedResponse, ParsedResponseRecord, @@ -300,7 +300,7 @@ def _record(datas: list) -> "object": record = MagicMock(spec=ParsedResponseRecord) record.content_responses = [ - ParsedResponse(perf_ns=i, data=d) for i, d in enumerate(datas) + ParsedResponse(perf_ns=i, data=d) for i, d in enumerate(data_list) ] return record diff --git a/tests/unit/common/models/test_record_models.py b/tests/unit/common/models/test_record_models.py index 5878553802..96628f3161 100644 --- a/tests/unit/common/models/test_record_models.py +++ b/tests/unit/common/models/test_record_models.py @@ -141,6 +141,46 @@ def test_profile_results_multiple_timeslices_maps_metrics_by_tag(self) -> None: } +class TestRecordDataStrictRouting: + """RecordData routes by record_type and raises on an unregistered value. + + ``strict_routing=True`` replaces AutoRoutedModel's base-class fallback: the + base RecordData has no standalone shape, so an unknown record_type must fail + loudly instead of degrading to a bare instance with every typed field dropped. + """ + + def test_unknown_record_type_raises(self) -> None: + from aiperf.common.models.record_models import RecordData + + with pytest.raises(ValueError, match="Unknown record_type 'does_not_exist'"): + RecordData.from_json({"record_type": "does_not_exist"}) + + def test_registered_record_type_routes_to_subclass(self) -> None: + # Importing the module registers the subclass via __init_subclass__. + from aiperf.accuracy.models import AccuracyRecordsData + from aiperf.common.enums import CreditPhase + from aiperf.common.models.record_models import RecordData + + original = AccuracyRecordsData( + session_num=0, + worker_id="w1", + benchmark_phase=CreditPhase.PROFILING, + timestamp_ns=1_000, + grader_name="multiple_choice", + passed=True, + confidence=1.0, + expected="A", + actual="A", + explanation="ok", + ) + + routed = RecordData.from_json(original.model_dump()) + + assert isinstance(routed, AccuracyRecordsData) + assert routed.expected == "A" + assert routed.grader_name == "multiple_choice" + + class TestSSEMessageDataclass: """Test that SSEMessage dataclass works correctly.""" diff --git a/tests/unit/records/test_records_manager.py b/tests/unit/records/test_records_manager.py index 69ae468c8f..5fcef15349 100644 --- a/tests/unit/records/test_records_manager.py +++ b/tests/unit/records/test_records_manager.py @@ -345,7 +345,7 @@ async def test_on_credit_phase_start_dispatches_timing_snapshot(self) -> None: await manager._on_credit_phase_start(message) manager._records_tracker.update_phase_info.assert_called_once_with(stats) - manager._dispatch_record.assert_awaited_once_with(stats) + manager._dispatch_record.assert_awaited_once_with(stats, warn_if_unrouted=False) @pytest.mark.asyncio async def test_on_credit_phase_progress_dispatches_timing_snapshot(self) -> None: @@ -357,7 +357,7 @@ async def test_on_credit_phase_progress_dispatches_timing_snapshot(self) -> None ) manager._records_tracker.update_phase_info.assert_called_once_with(stats) - manager._dispatch_record.assert_awaited_once_with(stats) + manager._dispatch_record.assert_awaited_once_with(stats, warn_if_unrouted=False) @pytest.mark.asyncio async def test_on_credit_phase_sending_complete_dispatches_timing_snapshot( @@ -376,7 +376,7 @@ async def test_on_credit_phase_sending_complete_dispatches_timing_snapshot( ) manager._records_tracker.update_phase_info.assert_called_once_with(stats) - manager._dispatch_record.assert_awaited_once_with(stats) + manager._dispatch_record.assert_awaited_once_with(stats, warn_if_unrouted=False) @pytest.mark.asyncio async def test_on_credit_phase_complete_dispatches_timing_snapshot(self) -> None: @@ -395,7 +395,7 @@ async def test_on_credit_phase_complete_dispatches_timing_snapshot(self) -> None ) manager._records_tracker.update_phase_info.assert_called_once_with(stats) - manager._dispatch_record.assert_awaited_once_with(stats) + manager._dispatch_record.assert_awaited_once_with(stats, warn_if_unrouted=False) @pytest.mark.asyncio async def test_on_metric_records_records_complete_before_phase_complete_defers_finalization( @@ -558,7 +558,7 @@ async def test_finalization_runs_when_final_record_arrives_during_phase_complete timing_dispatch_started = asyncio.Event() release_timing_dispatch = asyncio.Event() - async def _block_timing_dispatch(record) -> list[BaseException]: + async def _block_timing_dispatch(record, **_kwargs) -> list[BaseException]: if isinstance(record, CreditPhaseStats): timing_dispatch_started.set() await release_timing_dispatch.wait() diff --git a/tests/unit/records/test_records_manager_routing.py b/tests/unit/records/test_records_manager_routing.py index 59ff05b90e..eb1d8b2037 100644 --- a/tests/unit/records/test_records_manager_routing.py +++ b/tests/unit/records/test_records_manager_routing.py @@ -237,6 +237,19 @@ async def test_dispatch_with_no_handlers_warns_once(self) -> None: manager.warning.assert_called_once() manager.error.assert_not_called() + @pytest.mark.asyncio + async def test_dispatch_unrouted_control_plane_does_not_warn(self) -> None: + # Control-plane records (e.g. credit_phase_stats) are consumed elsewhere + # and only OPTIONALLY streamed, so an absent handler is expected -- passing + # warn_if_unrouted=False must suppress the misleading "records dropped" warning. + manager = self._manager({}) + record = MagicMock(record_type="credit_phase_stats") + + errors = await manager._dispatch_record(record, warn_if_unrouted=False) + + assert errors == [] + manager.warning.assert_not_called() + @pytest.mark.asyncio async def test_dispatch_missing_record_type_returns_error(self) -> None: manager = self._manager({}) @@ -264,6 +277,25 @@ async def test_dispatch_handler_exception_logged_and_returned(self) -> None: manager.error.assert_called_once() assert "boom" in manager.error.call_args[0][0] + @pytest.mark.asyncio + async def test_dispatch_best_effort_handler_exception_not_counted(self) -> None: + # A best-effort handler (streaming telemetry like OTel/MLflow) that raises + # must be logged but NOT counted -- a downed collector is not an inference + # failure and must not pollute the benchmark's phase error summary. + best_effort = StubStreamExporter() + best_effort.is_best_effort = True + best_effort.process_record.side_effect = RuntimeError("collector down") + strict = StubStreamExporter() + strict.process_record.side_effect = ValueError("real failure") + manager = self._manager({"metric_records": [best_effort, strict]}) + + errors = await manager._dispatch_record(MagicMock(record_type="metric_records")) + + # Only the strict handler's failure is returned; both are logged. + assert len(errors) == 1 + assert isinstance(errors[0], ValueError) + assert manager.error.call_count == 2 + @pytest.mark.asyncio async def test_dispatch_multiple_handler_exceptions(self) -> None: acc = StubAccumulator() From 90b1d4174548f325c2c23afd54dc314be47289c0 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 16 Jul 2026 08:13:26 -0700 Subject: [PATCH 34/39] fix(metrics): emit zero-valued per-watt metrics; normalize unbounded server-metrics window Two findings from the PR review pass: - energy_efficiency_analyzer._per_watt_metrics compared throughput/goodput by truthiness, so a genuine 0.0 (e.g. goodput_per_watt == 0 when no request meets the SLO) was silently dropped. Compare to None instead; avg_power_w > 0 is already guaranteed so the divisions stay well-defined. - server_metrics accumulator.export_results passed ExportContext's Optional start_ns/end_ns straight into the int-only max()/comparison in _compute_endpoint_summaries. Production callers always pass ints, but a bare ExportContext() (None bounds) would raise TypeError; normalize None -> 0 (unbounded), matching the GPU accumulator's None handling. Add regression tests for both. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- .../metrics/energy_efficiency_analyzer.py | 13 ++++++-- src/aiperf/server_metrics/accumulator.py | 10 +++++-- .../test_energy_efficiency_analyzer.py | 19 ++++++++++++ tests/unit/server_metrics/test_accumulator.py | 30 +++++++++++++++++++ 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/src/aiperf/metrics/energy_efficiency_analyzer.py b/src/aiperf/metrics/energy_efficiency_analyzer.py index 4419b0714c..3057c1d1dc 100644 --- a/src/aiperf/metrics/energy_efficiency_analyzer.py +++ b/src/aiperf/metrics/energy_efficiency_analyzer.py @@ -239,12 +239,19 @@ def _per_watt_metrics( if avg_power_w <= 0: return [] out: list[MetricResult] = [] - if throughput := metric("request_throughput"): + # Compare to None, not truthiness: a genuine 0.0 throughput/goodput is a + # valid per-watt result (e.g. goodput_per_watt == 0 when no request met the + # SLO) and must be emitted, not silently dropped. avg_power_w > 0 is + # guaranteed above, so none of these divisions is degenerate. + throughput = metric("request_throughput") + if throughput is not None: out.append(_result(PerformancePerWattMetric, throughput / avg_power_w)) - if output_tps := metric("output_token_throughput"): + output_tps = metric("output_token_throughput") + if output_tps is not None: out.append( _result(OutputTokensPerSecondPerWattMetric, output_tps / avg_power_w) ) - if goodput := metric("goodput"): + goodput = metric("goodput") + if goodput is not None: out.append(_result(GoodputPerWattMetric, goodput / avg_power_w)) return out diff --git a/src/aiperf/server_metrics/accumulator.py b/src/aiperf/server_metrics/accumulator.py index 7a5abebcf6..efa1cd282c 100644 --- a/src/aiperf/server_metrics/accumulator.py +++ b/src/aiperf/server_metrics/accumulator.py @@ -146,8 +146,14 @@ async def export_results(self, ctx: ExportContext) -> ServerMetricsResults | Non ServerMetricsResults containing endpoint summaries with computed statistics, or None if no endpoints were successfully scraped during profiling. """ - start_ns = ctx.start_ns - end_ns = ctx.end_ns + # ExportContext bounds are Optional (None = unbounded). Production callers + # always pass concrete ints, but normalize here so a bare + # ``export_results(ExportContext())`` can't reach the int-only max()/ + # comparison in _compute_endpoint_summaries and raise TypeError. 0 means + # "from the beginning"; the per-endpoint max(end, last_update) still + # captures the final scrape. + start_ns = ctx.start_ns or 0 + end_ns = ctx.end_ns or 0 error_summary = ctx.error_summary warmup_start_ns = ctx.warmup_start_ns warmup_end_ns = ctx.warmup_end_ns diff --git a/tests/unit/metrics/test_energy_efficiency_analyzer.py b/tests/unit/metrics/test_energy_efficiency_analyzer.py index b84febd413..6e0d1dbe23 100644 --- a/tests/unit/metrics/test_energy_efficiency_analyzer.py +++ b/tests/unit/metrics/test_energy_efficiency_analyzer.py @@ -105,6 +105,25 @@ async def test_full_metric_set_matches_doc_formulas(self) -> None: assert by_tag["output_tps_per_watt"] == pytest.approx(0.5) # 50 / 100 assert by_tag["goodput_per_watt"] == pytest.approx(0.08) # 8 / 100 + @pytest.mark.asyncio + async def test_zero_valued_per_watt_metrics_are_emitted_not_dropped(self) -> None: + # A genuine 0.0 goodput (no request met the SLO) is a valid measurement: + # goodput_per_watt must be emitted as 0.0, not silently omitted. + gpu = _StubGpu(energy=(1000.0, 2), power=(200.0, 2)) + summary = _metrics_summary( + total_osl=5000.0, + request_count=100.0, + request_throughput=10.0, + output_token_throughput=50.0, + goodput=0.0, + ) + + results = await _analyzer(concurrency=8).analyze(_ctx(gpu, summary)) + by_tag = {r.tag: r.avg for r in results} + + assert "goodput_per_watt" in by_tag + assert by_tag["goodput_per_watt"] == pytest.approx(0.0) + @pytest.mark.asyncio async def test_no_gpu_accumulator_returns_empty(self) -> None: summary = _metrics_summary(total_osl=5000.0) diff --git a/tests/unit/server_metrics/test_accumulator.py b/tests/unit/server_metrics/test_accumulator.py index e30bebed93..b9c3833352 100644 --- a/tests/unit/server_metrics/test_accumulator.py +++ b/tests/unit/server_metrics/test_accumulator.py @@ -152,6 +152,36 @@ async def test_export_results_with_data( assert result.endpoint_summaries is not None assert len(result.endpoint_summaries) == 1 + async def test_export_results_unbounded_context_does_not_crash( + self, + mock_cfg: BenchmarkRun, + ) -> None: + """A bare ExportContext() (start_ns/end_ns == None) must not reach the + int-only max()/comparison in _compute_endpoint_summaries and raise; None + bounds are normalized to unbounded and still produce a summary.""" + processor = ServerMetricsAccumulator(mock_cfg) + for i in range(3): + gauge = MetricFamily( + type=PrometheusMetricType.GAUGE, + description="KV cache usage", + samples=[MetricSample(labels=None, value=0.4 + i * 0.05)], + ) + await processor.process_server_metrics_record( + ServerMetricsRecord( + endpoint_url="http://node1:8081/metrics", + timestamp_ns=1_000_000_000 + i * 100_000_000, + endpoint_latency_ns=5_000_000, + metrics={"cache_usage": gauge}, + ) + ) + + result = await processor.export_results(ExportContext()) + + assert result is not None + assert isinstance(result, ServerMetricsResults) + assert result.endpoint_summaries is not None + assert len(result.endpoint_summaries) == 1 + async def test_export_results_includes_warmup_endpoint_summaries( self, mock_cfg: BenchmarkRun, From 32019296e9228c21b4906af8892058159b4485eb Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 16 Jul 2026 08:17:29 -0700 Subject: [PATCH 35/39] fix(models): evaluate strict routing even when the lookup table is empty The strict-routing raise was nested under 'if cls_discriminator and cls._model_lookup_table', so an empty lookup table (producing module not imported yet) skipped the check and silently degraded to cls.model_validate on the base -- defeating strict_routing's purpose. Enter the routing block when the table is non-empty OR the hierarchy is strict, so an unregistered type raises as promised. Non-strict hierarchies (e.g. the command tree) are unchanged. Add a regression test with an empty lookup table. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/common/models/auto_routed_model.py | 7 +++++-- tests/unit/common/models/test_record_models.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/aiperf/common/models/auto_routed_model.py b/src/aiperf/common/models/auto_routed_model.py index 57b2e7a4f0..d69d2b866c 100644 --- a/src/aiperf/common/models/auto_routed_model.py +++ b/src/aiperf/common/models/auto_routed_model.py @@ -96,9 +96,12 @@ def from_json(cls, json_or_dict: str | bytes | bytearray | dict) -> Self: else load_json_str(json_or_dict) ) - # Only route if THIS class explicitly set a discriminator (not inherited) + # Only route if THIS class explicitly set a discriminator (not inherited). + # For strict hierarchies, enter even when the lookup table is empty: an + # empty table means the producing module isn't imported yet, and strict + # routing must still raise rather than silently degrade to the base. cls_discriminator = cls.__dict__.get("discriminator_field") - if cls_discriminator and cls._model_lookup_table: + if cls_discriminator and (cls._model_lookup_table or cls.strict_routing): discriminator_value = data.get(cls_discriminator) if not discriminator_value: diff --git a/tests/unit/common/models/test_record_models.py b/tests/unit/common/models/test_record_models.py index 96628f3161..b10baba061 100644 --- a/tests/unit/common/models/test_record_models.py +++ b/tests/unit/common/models/test_record_models.py @@ -180,6 +180,20 @@ def test_registered_record_type_routes_to_subclass(self) -> None: assert routed.expected == "A" assert routed.grader_name == "multiple_choice" + def test_strict_routing_raises_with_empty_lookup_table(self) -> None: + # Even when NO subclass is registered (producing module not imported yet), + # a strict hierarchy must raise rather than silently degrade to the base. + from aiperf.common.models.auto_routed_model import AutoRoutedModel + + class _StrictRoot(AutoRoutedModel): + discriminator_field = "kind" + strict_routing = True + kind: str + + assert _StrictRoot._model_lookup_table == {} # nothing registered + with pytest.raises(ValueError, match="Unknown kind 'nope'"): + _StrictRoot.from_json({"kind": "nope"}) + class TestSSEMessageDataclass: """Test that SSEMessage dataclass works correctly.""" From 4785b670520d658029c4febe46a2c6ad8359a623 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 16 Jul 2026 08:25:27 -0700 Subject: [PATCH 36/39] docs/types: address PR review conventions (field descriptions, typed contracts, doc fixes) Non-functional review cleanups: - Add Field(description=) to the serialized record_type discriminators on MetricRecordsData and AccuracyRecordsData (repo standard: every Pydantic field documented). Discriminator registration unchanged (AutoRoutedModel reads the FieldInfo default). - Type the producer/observer contract: RecordProcessorProtocol.process_record returns RecordData | None, and RecordObserverContext.produced/get() are typed as RecordData (was Any); fix the docstring example (MetricRecordsData, not MetricRecordDict). - Add field docstrings to LoadedAnalyzer. - Refresh the _AccuracySettings docstring (now also covers the cancel-wait timeout) and regenerate environment-variables.md. - Docs: route the accuracy architecture diagram through RecordsManager; drop a duplicate concurrency-denominator bullet in metrics-reference. - Tests: top-level imports + concrete annotations for the accuracy record helper, annotate _make_processor_mock, and underscore-prefix unused p50/p90. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/accuracy/accuracy-benchmarking.md | 5 +++-- docs/environment-variables.md | 2 +- docs/metrics-reference.md | 3 +-- src/aiperf/accuracy/models.py | 6 +++++- src/aiperf/common/environment.py | 6 +++--- src/aiperf/common/messages/inference_messages.py | 6 +++++- src/aiperf/post_processors/protocols.py | 6 +++--- src/aiperf/post_processors/record_observer_context.py | 9 +++++---- src/aiperf/records/records_manager_processing.py | 7 +++++++ tests/unit/accuracy/test_accuracy_record_processor.py | 8 ++------ tests/unit/metrics/test_replay_sched_lag_metrics.py | 2 +- tests/unit/records/test_record_processor_service.py | 5 ++++- 12 files changed, 40 insertions(+), 25 deletions(-) diff --git a/docs/accuracy/accuracy-benchmarking.md b/docs/accuracy/accuracy-benchmarking.md index e6386a9c49..817ec7a9e6 100644 --- a/docs/accuracy/accuracy-benchmarking.md +++ b/docs/accuracy/accuracy-benchmarking.md @@ -437,8 +437,9 @@ reasoning model thought before an `unparsed` answer. ```mermaid flowchart LR DL[AccuracyDatasetLoader] -->|Conversation/Turn objects| RP[AccuracyRecordProcessor
grades each response] - RP -->|AccuracyRecordsData
on the accuracy channel| ACC[AccuracyAccumulator
per-task AccuracySummary] - RP -->|AccuracyRecordsData| JW[AccuracyJSONLWriter
accuracy_export.jsonl] + RP -->|AccuracyRecordsData
in RecordsMessage| RM[RecordsManager
metadata-driven routing] + RM --> ACC[AccuracyAccumulator
per-task AccuracySummary] + RM --> JW[AccuracyJSONLWriter
accuracy_export.jsonl] ACC --> CE[AccuracyConsoleExporter
Rich table] ACC --> DE[AccuracyDataExporter
accuracy_results.csv] ``` diff --git a/docs/environment-variables.md b/docs/environment-variables.md index c37aac0a0d..2a58f8f33e 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -32,7 +32,7 @@ CLI runner post-run callback behavior. Controls whether OnComplete callback exce ## ACCURACY -Accuracy benchmark settings. Tunables for the accuracy benchmark loaders. Currently only pins the LiveCodeBench dataset release so accuracy numbers are reproducible across runs without requiring source edits. +Accuracy benchmark settings. Tunables for accuracy benchmarking: the cancel-path result-wait timeout and the LiveCodeBench dataset release pin, so accuracy behavior and numbers are reproducible across runs without requiring source edits. | Environment Variable | Default | Constraints | Description | |----------------------|---------|-------------|-------------| diff --git a/docs/metrics-reference.md b/docs/metrics-reference.md index b537e1a491..36f377af4f 100644 --- a/docs/metrics-reference.md +++ b/docs/metrics-reference.md @@ -1970,8 +1970,7 @@ energy_per_user_j = total_gpu_energy / concurrency **Notes:** - Unit: `joules/user`. - Flagged `MetricFlags.NONE` — smaller-is-better is the default for unflagged metrics. -- Denominator is the profiling phase's configured `concurrency`. The resolver defaults this to `1` when `--concurrency` isn't specified in concurrency-mode runs, so the metric is emitted in the common case. -- Denominator is the profiling phase's configured `concurrency` (`run.cfg.get_profiling_phases()[0].concurrency`). +- Denominator is the profiling phase's configured `concurrency` (`run.cfg.get_profiling_phases()[0].concurrency`). The resolver defaults this to `1` when `--concurrency` isn't specified in concurrency-mode runs, so the metric is emitted in the common case. - Omitted when concurrency is unset (e.g. pure `--request-rate` mode) or aggregate GPU energy is unavailable. --- diff --git a/src/aiperf/accuracy/models.py b/src/aiperf/accuracy/models.py index 07ca452d5f..d0e651b25d 100644 --- a/src/aiperf/accuracy/models.py +++ b/src/aiperf/accuracy/models.py @@ -91,7 +91,11 @@ class AccuracyRecordsData(RecordData): boundary; the routing layer still reads it via ``getattr(record, "record_type")``. """ - record_type: Literal["accuracy"] = "accuracy" + record_type: Literal["accuracy"] = Field( + default="accuracy", + description="Serialized discriminator routing this record to the accuracy " + "channel for wire reconstruction.", + ) session_num: int = Field( ge=0, diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index cd904480c9..31f32f9e43 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -64,9 +64,9 @@ class _AccuracySettings(BaseSettings): """Accuracy benchmark settings. - Tunables for the accuracy benchmark loaders. Currently only pins the - LiveCodeBench dataset release so accuracy numbers are reproducible - across runs without requiring source edits. + Tunables for accuracy benchmarking: the cancel-path result-wait timeout and + the LiveCodeBench dataset release pin, so accuracy behavior and numbers are + reproducible across runs without requiring source edits. """ model_config = SettingsConfigDict(env_prefix="AIPERF_ACCURACY_") diff --git a/src/aiperf/common/messages/inference_messages.py b/src/aiperf/common/messages/inference_messages.py index a6c75b17e6..0d988f64c3 100644 --- a/src/aiperf/common/messages/inference_messages.py +++ b/src/aiperf/common/messages/inference_messages.py @@ -26,7 +26,11 @@ class InferenceResultsMessage(BaseServiceMessage): class MetricRecordsData(RecordData): """Incoming data from the record processor service to combine metric records for the profile run.""" - record_type: Literal["metric_records"] = "metric_records" + record_type: Literal["metric_records"] = Field( + default="metric_records", + description="Serialized discriminator routing this record to the " + "metric_records channel for wire reconstruction.", + ) metadata: MetricRecordMetadata = Field( ..., description="The metadata of the request record." diff --git a/src/aiperf/post_processors/protocols.py b/src/aiperf/post_processors/protocols.py index 098d04083d..7e8bf0d3e7 100644 --- a/src/aiperf/post_processors/protocols.py +++ b/src/aiperf/post_processors/protocols.py @@ -3,13 +3,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Protocol, runtime_checkable from aiperf.common.models import ParsedResponseRecord from aiperf.common.protocols import AIPerfLifecycleProtocol if TYPE_CHECKING: - from aiperf.common.models.record_models import MetricRecordMetadata + from aiperf.common.models.record_models import MetricRecordMetadata, RecordData from aiperf.post_processors.record_observer_context import RecordObserverContext @@ -20,7 +20,7 @@ class RecordProcessorProtocol(AIPerfLifecycleProtocol, Protocol): async def process_record( self, record: ParsedResponseRecord, metadata: MetricRecordMetadata - ) -> Any: ... + ) -> RecordData | None: ... @runtime_checkable diff --git a/src/aiperf/post_processors/record_observer_context.py b/src/aiperf/post_processors/record_observer_context.py index bc5769210f..6455d35211 100644 --- a/src/aiperf/post_processors/record_observer_context.py +++ b/src/aiperf/post_processors/record_observer_context.py @@ -5,12 +5,13 @@ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from aiperf.common.messages.inference_messages import MetricRecordsData if TYPE_CHECKING: from aiperf.common.models import MetricRecordMetadata, ParsedResponseRecord + from aiperf.common.models.record_models import RecordData # The record_type channel the metric producer declares in plugins.yaml, which is # also the key the RecordProcessorService groups its output under in ``produced``. @@ -29,15 +30,15 @@ class RecordObserverContext: metadata: MetricRecordMetadata """The metric record metadata for this record.""" - produced: dict[str, list[Any]] + produced: dict[str, list[RecordData]] """Producer outputs keyed by declared record_type. Read-only by contract: the wire payload is snapshotted before observers run, so mutating this cannot change what RecordsManager ingests. - e.g. ``{"metric_records": [MetricRecordDict], "accuracy": [AccuracyRecordsData]}``. + e.g. ``{"metric_records": [MetricRecordsData], "accuracy": [AccuracyRecordsData]}``. """ - def get(self, record_type: str) -> list[Any]: + def get(self, record_type: str) -> list[RecordData]: """Return the producer outputs emitted on ``record_type`` (empty if none).""" return self.produced.get(record_type, []) diff --git a/src/aiperf/records/records_manager_processing.py b/src/aiperf/records/records_manager_processing.py index 73f44a774f..b84e6063c4 100644 --- a/src/aiperf/records/records_manager_processing.py +++ b/src/aiperf/records/records_manager_processing.py @@ -50,8 +50,15 @@ class LoadedAnalyzer: """ analyzer: AnalyzerProtocol + """The instantiated analyzer plugin.""" + required_accumulators: list[str] = field(default_factory=list) + """AccumulatorType names whose LIVE instance the analyzer queries via + ``SummaryContext.get_accumulator``.""" + required_summaries: list[str] = field(default_factory=list) + """AccumulatorType names whose SUMMARY output the analyzer reads via + ``SummaryContext.get_output``.""" class _LoaderHost(Protocol): diff --git a/tests/unit/accuracy/test_accuracy_record_processor.py b/tests/unit/accuracy/test_accuracy_record_processor.py index c58c16706f..35ccd8fba3 100644 --- a/tests/unit/accuracy/test_accuracy_record_processor.py +++ b/tests/unit/accuracy/test_accuracy_record_processor.py @@ -9,6 +9,7 @@ from aiperf.accuracy.models import AccuracyRecordsData, GradingResult from aiperf.common.enums import CreditPhase from aiperf.common.models.dataset_models import ConversationMetadata, DatasetMetadata +from aiperf.common.models.record_models import ParsedResponse, ParsedResponseRecord from aiperf.config import BenchmarkRun from aiperf.plugin.enums import ( AccuracyBenchmarkType, @@ -292,12 +293,7 @@ class TestExtractOutputAndThinking: """`_extract_output_and_thinking` splits answer content from reasoning.""" @staticmethod - def _record(data_list: list) -> "object": - from aiperf.common.models.record_models import ( - ParsedResponse, - ParsedResponseRecord, - ) - + def _record(data_list: list[str]) -> ParsedResponseRecord: record = MagicMock(spec=ParsedResponseRecord) record.content_responses = [ ParsedResponse(perf_ns=i, data=d) for i, d in enumerate(data_list) diff --git a/tests/unit/metrics/test_replay_sched_lag_metrics.py b/tests/unit/metrics/test_replay_sched_lag_metrics.py index 07c33afe86..cd01dae5ac 100644 --- a/tests/unit/metrics/test_replay_sched_lag_metrics.py +++ b/tests/unit/metrics/test_replay_sched_lag_metrics.py @@ -186,7 +186,7 @@ def test_warn_degraded_called_once_with_percentiles(self) -> None: assert results[ReplaySchedDegradedMetric.tag].avg == 1.0 assert len(calls) == 1 - p50, p90, p99 = calls[0] + _p50, _p90, p99 = calls[0] assert p99 == pytest.approx(600.0) def test_warn_degraded_not_called_when_healthy(self) -> None: diff --git a/tests/unit/records/test_record_processor_service.py b/tests/unit/records/test_record_processor_service.py index 7d0bfa9742..6a5b6a0560 100644 --- a/tests/unit/records/test_record_processor_service.py +++ b/tests/unit/records/test_record_processor_service.py @@ -35,7 +35,10 @@ def _make_accuracy_record(session_num: int = 0) -> AccuracyRecordsData: ) -def _make_processor_mock(producers, observers=None): +def _make_processor_mock( + producers: list[tuple[str, MagicMock]], + observers: list[MagicMock] | None = None, +) -> MagicMock: """Build a RecordProcessor MagicMock wired for the 2-stage generic ship.""" mock_self = MagicMock(spec=RecordProcessor) mock_self.service_id = "rp" From 2a895276896a5ff72e891a8b050d64acb737714c Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 16 Jul 2026 08:35:13 -0700 Subject: [PATCH 37/39] fix(controller): watchdog to bound the normal-completion side-channel result wait The normal-completion shutdown gate is event-driven: it only re-checks readiness when a side-channel result message (telemetry / server metrics / accuracy) arrives. If one is never published -- e.g. RecordsManager's accuracy publish raises on a dead bus -- the corresponding gate never clears and shutdown hangs forever (only the cancel path had a bounded wait; accuracy has no status handshake so it's the most exposed). Arm a one-shot watchdog once profile results arrive: after Environment.SERVICE.SIDE_CHANNEL_RESULT_WAIT_SEC (default 30s) it force-clears any still-pending gates, logs which timed out, and re-triggers the shutdown check so export proceeds without the missing section rather than stalling. The watchdog is cancelled when all gates clear normally. Covers all three side channels. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/environment-variables.md | 1 + src/aiperf/common/environment.py | 10 +++ src/aiperf/controller/system_controller.py | 51 +++++++++++++++ .../unit/controller/test_system_controller.py | 63 +++++++++++++++++++ 4 files changed, 125 insertions(+) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 2a58f8f33e..871f1d4096 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -231,6 +231,7 @@ Service lifecycle and inter-service communication configuration. Controls timeou | `AIPERF_SERVICE_COMMS_REQUEST_TIMEOUT` | `90.0` | ≥ 1.0, ≤ 1000.0 | Timeout in seconds for requests from req_clients to rep_clients | | `AIPERF_SERVICE_CONNECTION_PROBE_INTERVAL` | `0.1` | ≥ 0.1, ≤ 600.0 | Interval in seconds for connection probes while waiting for initial connection to the zmq message bus | | `AIPERF_SERVICE_CONNECTION_PROBE_TIMEOUT` | `90.0` | ≥ 1.0, ≤ 100000.0 | Maximum time in seconds to wait for connection probe response while waiting for initial connection to the zmq message bus | +| `AIPERF_SERVICE_SIDE_CHANNEL_RESULT_WAIT_SEC` | `30.0` | ≥ 0.0, ≤ 100000.0 | Maximum time in seconds the SystemController waits, after the profile results arrive, for the side-channel results (telemetry, server metrics, accuracy) before proceeding to export without the missing ones. Bounds the normal-completion shutdown so a side-channel result message that is never published (e.g. a failed publish) cannot hang the run. | | `AIPERF_SERVICE_CREDIT_PROGRESS_REPORT_INTERVAL` | `2.0` | ≥ 1, ≤ 100000.0 | Interval in seconds between credit progress report messages | | `AIPERF_SERVICE_DISABLE_UVLOOP` | `False` | — | Disable uvloop and use default asyncio event loop instead | | `AIPERF_SERVICE_HEARTBEAT_INTERVAL` | `5.0` | ≥ 1.0, ≤ 100000.0 | Interval in seconds between heartbeat messages for component services | diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index 31f32f9e43..9fd209505a 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -927,6 +927,16 @@ class _ServiceSettings(BaseSettings): default=90.0, description="Maximum time in seconds to wait for connection probe response while waiting for initial connection to the zmq message bus", ) + SIDE_CHANNEL_RESULT_WAIT_SEC: float = Field( + ge=0.0, + le=100000.0, + default=30.0, + description="Maximum time in seconds the SystemController waits, after the " + "profile results arrive, for the side-channel results (telemetry, server " + "metrics, accuracy) before proceeding to export without the missing ones. " + "Bounds the normal-completion shutdown so a side-channel result message " + "that is never published (e.g. a failed publish) cannot hang the run.", + ) CREDIT_PROGRESS_REPORT_INTERVAL: float = Field( ge=1, le=100000.0, diff --git a/src/aiperf/controller/system_controller.py b/src/aiperf/controller/system_controller.py index 510ba5c772..8bd9ea3fb9 100644 --- a/src/aiperf/controller/system_controller.py +++ b/src/aiperf/controller/system_controller.py @@ -174,6 +174,10 @@ def __init__( self._shutdown_triggered = False self._shutdown_lock = asyncio.Lock() + # Bounds the normal-completion wait on the side-channel result gates + # (telemetry / server metrics / accuracy) so a result message that is + # never published can't hang shutdown. Armed once profile results arrive. + self._result_watchdog_task: asyncio.Task | None = None self._api_enabled = False self._telemetry_endpoints_configured: list[str] = [] self._telemetry_endpoints_reachable: list[str] = [] @@ -641,6 +645,8 @@ async def _on_process_records_result_message( ) self._profile_results_received = True + # Arm the watchdog now that we're waiting on the side-channel gates. + self._arm_result_watchdog() # Coordinate with telemetry results before shutdown await self._check_and_trigger_shutdown() @@ -765,6 +771,48 @@ def _is_api_service_alive(self) -> bool: for rec in mp_info ) + def _arm_result_watchdog(self) -> None: + """Start the side-channel result watchdog exactly once. + + The normal-completion shutdown gate is event-driven: it only re-checks + readiness when a side-channel result message arrives. If one is never + published (e.g. RecordsManager's accuracy/telemetry/server-metrics publish + raises), the corresponding gate never clears and shutdown hangs forever. + This bounds that wait. + """ + if self._result_watchdog_task is not None or self._shutdown_triggered: + return + self._result_watchdog_task = asyncio.create_task(self._result_watchdog()) + + async def _result_watchdog(self) -> None: + """Force-clear still-pending side-channel gates after a bounded wait.""" + timeout = Environment.SERVICE.SIDE_CHANNEL_RESULT_WAIT_SEC + await asyncio.sleep(timeout) + if self._shutdown_triggered: + return + pending = [] + if self._should_wait_for_telemetry and self._telemetry_results is None: + pending.append("telemetry") + if ( + self._should_wait_for_server_metrics + and self._server_metrics_results is None + ): + pending.append("server metrics") + if self._should_wait_for_accuracy and self._accuracy_results is None: + pending.append("accuracy") + if not pending: + return + self.warning( + f"Side-channel results did not arrive within {timeout}s of the profile " + f"results: {', '.join(pending)}. Proceeding to export without them " + "(their result message was likely never published)." + ) + # Release only the gates that timed out; ones that arrived keep their data. + self._should_wait_for_telemetry = False + self._should_wait_for_server_metrics = False + self._should_wait_for_accuracy = False + await self._check_and_trigger_shutdown() + async def _check_and_trigger_shutdown(self) -> None: """Check if all required results are received and trigger unified export + shutdown. @@ -822,6 +870,9 @@ async def _check_and_trigger_shutdown(self) -> None: ): self._shutdown_triggered = True should_shutdown = True + # The gates are satisfied; the watchdog is no longer needed. + if self._result_watchdog_task is not None: + self._result_watchdog_task.cancel() self.info("All results received, initiating shutdown") else: if not telemetry_ready_for_shutdown: diff --git a/tests/unit/controller/test_system_controller.py b/tests/unit/controller/test_system_controller.py index ecbbb0481a..6c49ae08f8 100644 --- a/tests/unit/controller/test_system_controller.py +++ b/tests/unit/controller/test_system_controller.py @@ -703,3 +703,66 @@ async def test_extends_grace_when_api_process_alive_and_running( monkeypatch.setattr(Environment.API_SERVER, "POST_COMPLETE_GRACE", 7.0) sleeps = await self._drive_stop(system_controller, monkeypatch) assert sleeps[0] == 7.0 + + +class TestResultWatchdog: + """The side-channel result watchdog bounds the normal-completion shutdown so a + result message that is never published cannot hang the run.""" + + @staticmethod + def _controller() -> SystemController: + sc = SystemController.__new__(SystemController) + sc._shutdown_triggered = False + sc._should_wait_for_telemetry = False + sc._telemetry_results = None + sc._should_wait_for_server_metrics = False + sc._server_metrics_results = None + sc._should_wait_for_accuracy = False + sc._accuracy_results = None + sc.warning = MagicMock() + sc._check_and_trigger_shutdown = AsyncMock() + return sc + + @pytest.mark.asyncio + async def test_watchdog_releases_stalled_accuracy_gate(self) -> None: + sc = self._controller() + sc._should_wait_for_accuracy = True # message never arrived + + await sc._result_watchdog() + + assert sc._should_wait_for_accuracy is False + sc.warning.assert_called_once() + assert "accuracy" in sc.warning.call_args[0][0] + sc._check_and_trigger_shutdown.assert_awaited_once() + + @pytest.mark.asyncio + async def test_watchdog_noop_when_result_already_arrived(self) -> None: + sc = self._controller() + sc._should_wait_for_accuracy = True + sc._accuracy_results = object() # arrived before the timeout + + await sc._result_watchdog() + + sc.warning.assert_not_called() + sc._check_and_trigger_shutdown.assert_not_awaited() + + @pytest.mark.asyncio + async def test_watchdog_noop_when_shutdown_already_triggered(self) -> None: + sc = self._controller() + sc._should_wait_for_accuracy = True + sc._shutdown_triggered = True + + await sc._result_watchdog() + + sc.warning.assert_not_called() + sc._check_and_trigger_shutdown.assert_not_awaited() + + @pytest.mark.asyncio + async def test_arm_is_idempotent(self) -> None: + sc = self._controller() + sc._result_watchdog_task = None + sc._arm_result_watchdog() + first = sc._result_watchdog_task + sc._arm_result_watchdog() + assert sc._result_watchdog_task is first + first.cancel() From 53bbdb640700163b5913e9496dfbb59e0f0e6365 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 16 Jul 2026 08:39:10 -0700 Subject: [PATCH 38/39] fix(accuracy): make query_time_range sort-safe for interleaved arrivals AccuracyAccumulator.query_time_range used bisect on the append-ordered timestamp list, but records arrive interleaved from multiple record processors so the list is not globally sorted -- a binary search slices wrong. Filter directly instead (matching the mask-based query_time_range on the GPU/server accumulators) and drop the now-unused _timestamps_ns/bisect. Add an out-of-order regression test. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- src/aiperf/accuracy/accumulator.py | 14 +++++--------- tests/unit/accuracy/test_accumulator.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/aiperf/accuracy/accumulator.py b/src/aiperf/accuracy/accumulator.py index 777f1f91fd..e277f59e11 100644 --- a/src/aiperf/accuracy/accumulator.py +++ b/src/aiperf/accuracy/accumulator.py @@ -3,7 +3,6 @@ from __future__ import annotations -import bisect from collections import defaultdict from typing import TYPE_CHECKING, Any @@ -50,22 +49,19 @@ def __init__(self, run: BenchmarkRun, **kwargs: Any) -> None: super().__init__(run=run, **kwargs) self.run = run self._records: list[AccuracyRecordsData] = [] - self._timestamps_ns: list[int] = [] async def process_record(self, record: AccuracyRecordsData) -> None: - """Append a graded record, preserving per-worker insertion order.""" + """Append a graded record in arrival order.""" self._records.append(record) - self._timestamps_ns.append(record.timestamp_ns) def query_time_range(self, start_ns: int, end_ns: int) -> list[AccuracyRecordsData]: """Return records whose ``timestamp_ns`` is in ``[start_ns, end_ns)``. - Analyzer query surface. Uses ``bisect`` on the append-ordered - ``_timestamps_ns`` (records arrive time-ordered per worker). + Analyzer query surface. Filters directly rather than bisecting: records + arrive interleaved from multiple record processors, so ``_records`` is not + globally sorted by ``timestamp_ns`` and a binary search would slice wrong. """ - lo = bisect.bisect_left(self._timestamps_ns, start_ns) - hi = bisect.bisect_left(self._timestamps_ns, end_ns) - return self._records[lo:hi] + return [r for r in self._records if start_ns <= r.timestamp_ns < end_ns] def _build_summary(self, phase: CreditPhase | None) -> AccuracySummary | None: """Roll scoped records into an ``AccuracySummary`` (None when empty). diff --git a/tests/unit/accuracy/test_accumulator.py b/tests/unit/accuracy/test_accumulator.py index 3fcae8180a..be29abcd39 100644 --- a/tests/unit/accuracy/test_accumulator.py +++ b/tests/unit/accuracy/test_accumulator.py @@ -187,3 +187,21 @@ async def test_query_time_range_slices_by_timestamp(self) -> None: sliced = acc.query_time_range(20, 40) assert [r.timestamp_ns for r in sliced] == [20, 30] + + async def test_query_time_range_handles_out_of_order_arrival(self) -> None: + # Records arrive interleaved from multiple record processors, so the + # backing list is NOT globally sorted by timestamp. A binary search would + # slice wrong here; the direct filter must still return every in-range row. + acc = _make_accumulator() + await _seed( + acc, + [ + _record(timestamp_ns=30), + _record(timestamp_ns=10), + _record(timestamp_ns=40), + _record(timestamp_ns=20), + ], + ) + + sliced = acc.query_time_range(20, 40) + assert sorted(r.timestamp_ns for r in sliced) == [20, 30] From e2245fda8c2815748b3ea3efde858262f979336e Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Thu, 16 Jul 2026 09:04:47 -0700 Subject: [PATCH 39/39] refactor(controller): shared shutdown barrier via ResultJoinCoordinator Replace the SystemController's per-domain shutdown coordination -- the _should_wait_for_telemetry/server_metrics/accuracy boolean flags, the sleep-based result watchdog, and the cancel-path poll loop -- with a single shared barrier (ResultJoinCoordinator, ported from ajc/phase-baseline-handshake). Mechanics: - Services advertise result domains via a result_producer: capability in RegisterServiceCommand (new ServiceCapability/baseline enums + extra_capabilities ClassVar on BaseComponentService). RecordsManager advertises 'profile' (+ 'accuracy' iff enabled), GPU telemetry 'telemetry', server metrics 'server_metrics'. - The controller registers each advertised domain, completes it when the result arrives, and drops it when the producer dies (service error) or reports itself disabled (telemetry/server-metrics status). Shutdown triggers when the barrier is ready -- one check, no boolean soup, no watchdog. - Cancel path now awaits an asyncio.Event via wait_for instead of polling. Liveness for a dead producer is handled by unregister_service; removes the SIDE_CHANNEL_RESULT_WAIT_SEC watchdog knob. Ports ResultJoinCoordinator + BaselineKind/ServiceCapability enums and their tests; rewires the accuracy shutdown-gate tests to the barrier model. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anthony Casagrande --- docs/environment-variables.md | 1 - src/aiperf/common/base_component_service.py | 8 +- src/aiperf/common/enums/__init__.py | 10 + src/aiperf/common/enums/baseline_enums.py | 45 ++++ src/aiperf/common/environment.py | 10 - .../common/messages/command_messages.py | 9 + .../controller/result_join_coordinator.py | 67 ++++++ src/aiperf/controller/system_controller.py | 212 ++++++------------ src/aiperf/gpu_telemetry/manager.py | 12 +- src/aiperf/records/records_manager.py | 8 + src/aiperf/server_metrics/manager.py | 14 +- .../controller/test_accuracy_shutdown_gate.py | 70 +++--- .../test_result_join_coordinator.py | 123 ++++++++++ .../unit/controller/test_system_controller.py | 63 ------ 14 files changed, 402 insertions(+), 250 deletions(-) create mode 100644 src/aiperf/common/enums/baseline_enums.py create mode 100644 src/aiperf/controller/result_join_coordinator.py create mode 100644 tests/unit/controller/test_result_join_coordinator.py diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 871f1d4096..2a58f8f33e 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -231,7 +231,6 @@ Service lifecycle and inter-service communication configuration. Controls timeou | `AIPERF_SERVICE_COMMS_REQUEST_TIMEOUT` | `90.0` | ≥ 1.0, ≤ 1000.0 | Timeout in seconds for requests from req_clients to rep_clients | | `AIPERF_SERVICE_CONNECTION_PROBE_INTERVAL` | `0.1` | ≥ 0.1, ≤ 600.0 | Interval in seconds for connection probes while waiting for initial connection to the zmq message bus | | `AIPERF_SERVICE_CONNECTION_PROBE_TIMEOUT` | `90.0` | ≥ 1.0, ≤ 100000.0 | Maximum time in seconds to wait for connection probe response while waiting for initial connection to the zmq message bus | -| `AIPERF_SERVICE_SIDE_CHANNEL_RESULT_WAIT_SEC` | `30.0` | ≥ 0.0, ≤ 100000.0 | Maximum time in seconds the SystemController waits, after the profile results arrive, for the side-channel results (telemetry, server metrics, accuracy) before proceeding to export without the missing ones. Bounds the normal-completion shutdown so a side-channel result message that is never published (e.g. a failed publish) cannot hang the run. | | `AIPERF_SERVICE_CREDIT_PROGRESS_REPORT_INTERVAL` | `2.0` | ≥ 1, ≤ 100000.0 | Interval in seconds between credit progress report messages | | `AIPERF_SERVICE_DISABLE_UVLOOP` | `False` | — | Disable uvloop and use default asyncio event loop instead | | `AIPERF_SERVICE_HEARTBEAT_INTERVAL` | `5.0` | ≥ 1.0, ≤ 100000.0 | Interval in seconds between heartbeat messages for component services | diff --git a/src/aiperf/common/base_component_service.py b/src/aiperf/common/base_component_service.py index 5aaa3af213..04f5eb6e0c 100644 --- a/src/aiperf/common/base_component_service.py +++ b/src/aiperf/common/base_component_service.py @@ -3,7 +3,7 @@ import asyncio import uuid -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar from aiperf.common.base_service import BaseService from aiperf.common.enums import CommandType, LifecycleState @@ -40,6 +40,11 @@ class BaseComponentService(BaseService): publishing the current state of the service to the system controller. """ + # Capability tags advertised to the SystemController at registration. Result + # producers override this (e.g. ``make_result_producer_capability("telemetry")``) + # so the controller can join their result on shutdown. + extra_capabilities: ClassVar[tuple[str, ...]] = () + def __init__( self, run: "BenchmarkRun", @@ -77,6 +82,7 @@ async def _register_service_on_start(self) -> None: # Target the system controller directly to avoid broadcasting to all services. target_service_type=ServiceType.SYSTEM_CONTROLLER, state=self.state, + capabilities=tuple(self.extra_capabilities), ) max_attempts = Environment.SERVICE.REGISTRATION_MAX_ATTEMPTS registration_interval = Environment.SERVICE.REGISTRATION_INTERVAL diff --git a/src/aiperf/common/enums/__init__.py b/src/aiperf/common/enums/__init__.py index 44ab321adc..ef0251a3f4 100644 --- a/src/aiperf/common/enums/__init__.py +++ b/src/aiperf/common/enums/__init__.py @@ -6,6 +6,12 @@ BasePydanticEnumInfo, CaseInsensitiveStrEnum, ) +from aiperf.common.enums.baseline_enums import ( + BaselineKind, + ServiceCapability, + make_result_producer_capability, + parse_result_producer_capability, +) from aiperf.common.enums.enums import ( AIPerfLogLevel, AudioFormat, @@ -86,6 +92,7 @@ "AudioFormat", "BaseMetricUnit", "BaseMetricUnitInfo", + "BaselineKind", "BasePydanticBackedStrEnum", "BasePydanticEnumInfo", "CaseInsensitiveStrEnum", @@ -145,6 +152,7 @@ "SSEFieldType", "ServerMetricsDiscoveryMode", "ServerMetricsFormat", + "ServiceCapability", "ServiceRegistrationStatus", "SweepMode", "SystemState", @@ -155,4 +163,6 @@ "VideoJobStatus", "VideoSynthType", "WorkerStatus", + "make_result_producer_capability", + "parse_result_producer_capability", ] diff --git a/src/aiperf/common/enums/baseline_enums.py b/src/aiperf/common/enums/baseline_enums.py new file mode 100644 index 0000000000..9698e3eb3e --- /dev/null +++ b/src/aiperf/common/enums/baseline_enums.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Enums for the phase baseline handshake. + +BaselineKind tags whether a baseline reading is taken before a phase starts +issuing credits (START) or after credits have drained (END). + +ServiceCapability is a generic capability tag advertised by services in their +RegisterServiceCommand; SystemController dispatches based on membership +(e.g. BASELINE_COLLECTOR services join the BaselineCoordinator's registered set). +""" + +from aiperf.common.enums.base_enums import CaseInsensitiveStrEnum + + +class BaselineKind(CaseInsensitiveStrEnum): + """Direction of a baseline reading relative to a phase.""" + + START = "start" + END = "end" + + +class ServiceCapability(CaseInsensitiveStrEnum): + """Capability tags a service may advertise at registration time.""" + + BASELINE_COLLECTOR = "baseline_collector" + RESULT_PRODUCER = "result_producer" + + +_RESULT_PRODUCER_PREFIX = f"{ServiceCapability.RESULT_PRODUCER}:" + + +def make_result_producer_capability(domain: str) -> str: + """Build a result-producer capability tag for a result domain.""" + + return f"{_RESULT_PRODUCER_PREFIX}{domain}" + + +def parse_result_producer_capability(capability: str) -> str | None: + """Return the result domain if capability is a result-producer tag.""" + + if not capability.startswith(_RESULT_PRODUCER_PREFIX): + return None + domain = capability.removeprefix(_RESULT_PRODUCER_PREFIX) + return domain or None diff --git a/src/aiperf/common/environment.py b/src/aiperf/common/environment.py index 9fd209505a..31f32f9e43 100644 --- a/src/aiperf/common/environment.py +++ b/src/aiperf/common/environment.py @@ -927,16 +927,6 @@ class _ServiceSettings(BaseSettings): default=90.0, description="Maximum time in seconds to wait for connection probe response while waiting for initial connection to the zmq message bus", ) - SIDE_CHANNEL_RESULT_WAIT_SEC: float = Field( - ge=0.0, - le=100000.0, - default=30.0, - description="Maximum time in seconds the SystemController waits, after the " - "profile results arrive, for the side-channel results (telemetry, server " - "metrics, accuracy) before proceeding to export without the missing ones. " - "Bounds the normal-completion shutdown so a side-channel result message " - "that is never published (e.g. a failed publish) cannot hang the run.", - ) CREDIT_PROGRESS_REPORT_INTERVAL: float = Field( ge=1, le=100000.0, diff --git a/src/aiperf/common/messages/command_messages.py b/src/aiperf/common/messages/command_messages.py index 0a943977c8..1fdde4076d 100644 --- a/src/aiperf/common/messages/command_messages.py +++ b/src/aiperf/common/messages/command_messages.py @@ -290,6 +290,15 @@ class RegisterServiceCommand(CommandMessage): ..., description="The type of the service to register" ) state: LifecycleState = Field(..., description="The current state of the service") + capabilities: tuple[str, ...] = Field( + default=(), + description=( + "Capability tags advertised by this service. SystemController " + "dispatches based on membership; e.g. a 'result_producer:' tag " + "registers the service as a required producer for that result domain " + "in the ResultJoinCoordinator. Unknown tags are ignored." + ), + ) class ProcessRecordsResponse(CommandSuccessResponse): diff --git a/src/aiperf/controller/result_join_coordinator.py b/src/aiperf/controller/result_join_coordinator.py new file mode 100644 index 0000000000..614a35023b --- /dev/null +++ b/src/aiperf/controller/result_join_coordinator.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tracks post-profile result producers until every registered result is joined.""" + +from __future__ import annotations + + +class ResultJoinCoordinator: + """Coordinates readiness across registered result-producing domains.""" + + def __init__(self) -> None: + self._required: dict[str, set[str]] = {} + self._completed: dict[str, set[str]] = {} + self._last_reported_pending: tuple[str, ...] = () + + @property + def ready(self) -> bool: + return self.pending_domains == () + + @property + def pending_domains(self) -> tuple[str, ...]: + return tuple( + sorted( + domain + for domain, required in self._required.items() + if required - self._completed.get(domain, set()) + ) + ) + + def register(self, domain: str, service_id: str) -> None: + self._required.setdefault(domain, set()).add(service_id) + + def unregister(self, domain: str, service_id: str) -> None: + required = self._required.get(domain) + if required is None: + return + + required.discard(service_id) + completed = self._completed.get(domain) + if completed is not None: + completed.discard(service_id) + + if not required: + self._required.pop(domain, None) + self._completed.pop(domain, None) + + def unregister_service(self, service_id: str) -> None: + for domain in tuple(self._required): + self.unregister(domain, service_id) + + def complete(self, domain: str, service_id: str) -> None: + if service_id not in self._required.get(domain, set()): + return + self._completed.setdefault(domain, set()).add(service_id) + + def complete_domain(self, domain: str) -> None: + required = self._required.get(domain) + if not required: + return + self._completed[domain] = set(required) + + def pending_domains_changed(self) -> tuple[str, ...] | None: + pending = self.pending_domains + if pending == self._last_reported_pending: + return None + self._last_reported_pending = pending + return pending diff --git a/src/aiperf/controller/system_controller.py b/src/aiperf/controller/system_controller.py index 8bd9ea3fb9..683ac0ecb6 100644 --- a/src/aiperf/controller/system_controller.py +++ b/src/aiperf/controller/system_controller.py @@ -22,6 +22,7 @@ LifecycleState, MessageType, ServiceRegistrationStatus, + parse_result_producer_capability, ) from aiperf.common.environment import Environment from aiperf.common.exceptions import LifecycleOperationError @@ -63,6 +64,7 @@ from aiperf.controller.controller_utils import print_exit_errors from aiperf.controller.protocols import ServiceManagerProtocol from aiperf.controller.proxy_manager import ProxyManager +from aiperf.controller.result_join_coordinator import ResultJoinCoordinator from aiperf.controller.system_mixins import SignalHandlerMixin from aiperf.credit.messages import CreditsCompleteMessage from aiperf.exporters.exporter_manager import ExporterManager @@ -163,21 +165,19 @@ def __init__( self._server_metrics_results: ServerMetricsResults | None = None self._accuracy_results: AccuracySummary | None = None self._accuracy_results_injected = False - self._profile_results_received = False - self._should_wait_for_telemetry = False - self._should_wait_for_server_metrics = False - # Accuracy has no manager status message; gate on config directly since - # RecordsManager publishes the summary iff accuracy is enabled. - self._should_wait_for_accuracy = ( - self.run.cfg.accuracy is not None and self.run.cfg.accuracy.enabled - ) + # Shared shutdown barrier: every result-producing service that advertises a + # ``result_producer:`` capability at registration joins here, and + # each domain must complete before shutdown. A producer that dies (service + # error) or reports itself disabled (telemetry/server-metrics status) + # unregisters, so it stops blocking. Replaces the per-gate boolean flags. + self._result_join_coordinator = ResultJoinCoordinator() + # Set when the accuracy result message lands (even with results=None). The + # cancel path awaits this to give a graded summary a bounded chance to + # arrive; the normal path relies on the barrier alone. + self._accuracy_result_arrived = asyncio.Event() self._shutdown_triggered = False self._shutdown_lock = asyncio.Lock() - # Bounds the normal-completion wait on the side-channel result gates - # (telemetry / server metrics / accuracy) so a result message that is - # never published can't hang shutdown. Armed once profile results arrive. - self._result_watchdog_task: asyncio.Task | None = None self._api_enabled = False self._telemetry_endpoints_configured: list[str] = [] self._telemetry_endpoints_reachable: list[str] = [] @@ -265,14 +265,12 @@ async def _start_services(self) -> None: await self.service_manager.run_service(ServiceType.GPU_TELEMETRY_MANAGER) else: self.info("GPU telemetry disabled via --no-gpu-telemetry") - self._should_wait_for_telemetry = False if self.run.cfg.server_metrics.enabled: self.debug("Starting optional ServerMetricsManager service") await self.service_manager.run_service(ServiceType.SERVER_METRICS_MANAGER) else: self.info("Server metrics disabled via --no-server-metrics") - self._should_wait_for_server_metrics = False if self.run.cfg.network_latency.should_probe: self.debug("Starting optional NetworkLatencyManager service") @@ -398,6 +396,14 @@ async def _handle_register_service_command( self.service_manager.service_map[message.service_type] = [] self.service_manager.service_map[message.service_type].append(service_info) + # Join every result domain this service advertises into the shutdown + # barrier. A telemetry/server-metrics producer may later report it is + # disabled via its status message, which unregisters the domain again. + for capability in message.capabilities: + domain = parse_result_producer_capability(capability) + if domain is not None: + self._result_join_coordinator.register(domain, message.service_id) + try: type_name = ServiceType(message.service_type).name.title().replace("_", " ") except (TypeError, ValueError): @@ -463,6 +469,10 @@ async def _process_service_error_message( service_id=message.service_id, ) ) + # A dead producer can no longer join its result domain; drop it from the + # barrier so shutdown isn't blocked forever, then re-check readiness. + self._result_join_coordinator.unregister_service(message.service_id) + await self._check_and_trigger_shutdown() @on_message(MessageType.STATUS) async def _process_status_message(self, message: StatusMessage) -> None: @@ -512,9 +522,11 @@ async def _on_telemetry_status_message( self._telemetry_endpoints_configured = message.endpoints_configured self._telemetry_endpoints_reachable = message.endpoints_reachable - self._should_wait_for_telemetry = message.enabled if not message.enabled: + # Advertised the capability at registration but won't actually produce; + # drop the domain from the barrier. + self._result_join_coordinator.unregister("telemetry", message.service_id) reason_msg = f": {message.reason}" if message.reason else "" self.info(f"DCGM telemetry skipped{reason_msg}") else: @@ -536,9 +548,11 @@ async def _on_server_metrics_status_message( self._server_metrics_endpoints_configured = message.endpoints_configured self._server_metrics_endpoints_reachable = message.endpoints_reachable - self._should_wait_for_server_metrics = message.enabled if not message.enabled: + self._result_join_coordinator.unregister( + "server_metrics", message.service_id + ) reason_msg = f" - {message.reason}" if message.reason else "" self.info(f"Server metrics disabled{reason_msg}") else: @@ -644,10 +658,8 @@ async def _on_process_records_result_message( f"Received process records result message with no records: {message.results.results}" ) - self._profile_results_received = True - # Arm the watchdog now that we're waiting on the side-channel gates. - self._arm_result_watchdog() - # Coordinate with telemetry results before shutdown + self._result_join_coordinator.complete_domain("profile") + # Coordinate with the remaining result domains before shutdown await self._check_and_trigger_shutdown() @on_message(MessageType.PROCESS_TELEMETRY_RESULT) @@ -681,7 +693,7 @@ async def _on_process_telemetry_result_message( except Exception as e: self.exception(f"Error processing telemetry results message: {e!r}") finally: - self._should_wait_for_telemetry = False + self._result_join_coordinator.complete_domain("telemetry") await self._check_and_trigger_shutdown() @on_message(MessageType.PROCESS_SERVER_METRICS_RESULT) @@ -721,7 +733,7 @@ async def _on_process_server_metrics_result_message( except Exception as e: self.exception(f"Error processing server metrics results message: {e!r}") finally: - self._should_wait_for_server_metrics = False + self._result_join_coordinator.complete_domain("server_metrics") await self._check_and_trigger_shutdown() @on_message(MessageType.PROCESS_ACCURACY_RESULT) @@ -734,7 +746,8 @@ async def _on_process_accuracy_result_message( except Exception as e: self.exception(f"Error processing accuracy results message: {e!r}") finally: - self._should_wait_for_accuracy = False + self._accuracy_result_arrived.set() + self._result_join_coordinator.complete_domain("accuracy") await self._check_and_trigger_shutdown() def _is_api_service_alive(self) -> bool: @@ -771,70 +784,27 @@ def _is_api_service_alive(self) -> bool: for rec in mp_info ) - def _arm_result_watchdog(self) -> None: - """Start the side-channel result watchdog exactly once. - - The normal-completion shutdown gate is event-driven: it only re-checks - readiness when a side-channel result message arrives. If one is never - published (e.g. RecordsManager's accuracy/telemetry/server-metrics publish - raises), the corresponding gate never clears and shutdown hangs forever. - This bounds that wait. - """ - if self._result_watchdog_task is not None or self._shutdown_triggered: - return - self._result_watchdog_task = asyncio.create_task(self._result_watchdog()) - - async def _result_watchdog(self) -> None: - """Force-clear still-pending side-channel gates after a bounded wait.""" - timeout = Environment.SERVICE.SIDE_CHANNEL_RESULT_WAIT_SEC - await asyncio.sleep(timeout) - if self._shutdown_triggered: - return - pending = [] - if self._should_wait_for_telemetry and self._telemetry_results is None: - pending.append("telemetry") - if ( - self._should_wait_for_server_metrics - and self._server_metrics_results is None - ): - pending.append("server metrics") - if self._should_wait_for_accuracy and self._accuracy_results is None: - pending.append("accuracy") - if not pending: - return - self.warning( - f"Side-channel results did not arrive within {timeout}s of the profile " - f"results: {', '.join(pending)}. Proceeding to export without them " - "(their result message was likely never published)." - ) - # Release only the gates that timed out; ones that arrived keep their data. - self._should_wait_for_telemetry = False - self._should_wait_for_server_metrics = False - self._should_wait_for_accuracy = False - await self._check_and_trigger_shutdown() - async def _check_and_trigger_shutdown(self) -> None: - """Check if all required results are received and trigger unified export + shutdown. - - Coordination logic: - 1. Always wait for profile results (ProcessRecordsResultMessage) - 2. If telemetry disabled OR telemetry results received → proceed - 3. If server metrics disabled OR server metrics results received → proceed - 4. Otherwise → wait (results arrive nearly simultaneously and will call this method again) - - Thread safety: - Uses self._shutdown_lock to prevent race conditions when ProcessRecordsResultMessage, - ProcessTelemetryResultMessage, and ProcessServerMetricsResultMessage arrive concurrently. - The lock ensures atomic check-and-set of _shutdown_triggered, preventing double-triggering of stop(). + """Trigger unified export + shutdown once every result domain has joined. + + Readiness is owned by the ``ResultJoinCoordinator`` shutdown barrier: each + result-producing service that advertised a ``result_producer:`` + capability at registration must ``complete_domain`` (its result arrived) or + drop out (service error / reported-disabled) before the barrier is + ``ready``. This is called on every result/status/error message so it + re-checks as domains complete. + + Thread safety: ``self._shutdown_lock`` makes the check-and-set of + ``_shutdown_triggered`` atomic against concurrently-arriving result + messages, preventing double-triggering of stop(). """ self.debug( - f"_check_and_trigger_shutdown: profile_received={self._profile_results_received}, " - f"wait_telemetry={self._should_wait_for_telemetry}, telemetry_results={self._telemetry_results is not None}, " - f"wait_server_metrics={self._should_wait_for_server_metrics}, server_metrics_results={self._server_metrics_results is not None}, " - f"wait_accuracy={self._should_wait_for_accuracy}, accuracy_results={self._accuracy_results is not None}, " - f"shutdown_triggered={self._shutdown_triggered}" + lambda: ( + "_check_and_trigger_shutdown: " + f"pending_domains={self._result_join_coordinator.pending_domains}, " + f"shutdown_triggered={self._shutdown_triggered}" + ) ) - # Check if we should trigger shutdown (with lock protection) should_shutdown = False async with self._shutdown_lock: if self._shutdown_triggered: @@ -843,44 +813,15 @@ async def _check_and_trigger_shutdown(self) -> None: ) return - if not self._profile_results_received: - self.debug( - "_check_and_trigger_shutdown: profile results not received yet" - ) - return - - telemetry_ready_for_shutdown = ( - not self._should_wait_for_telemetry - or self._telemetry_results is not None - ) - - server_metrics_ready_for_shutdown = ( - not self._should_wait_for_server_metrics - or self._server_metrics_results is not None - ) - - accuracy_ready_for_shutdown = ( - not self._should_wait_for_accuracy or self._accuracy_results is not None - ) - - if ( - telemetry_ready_for_shutdown - and server_metrics_ready_for_shutdown - and accuracy_ready_for_shutdown - ): + if self._result_join_coordinator.ready: self._shutdown_triggered = True should_shutdown = True - # The gates are satisfied; the watchdog is no longer needed. - if self._result_watchdog_task is not None: - self._result_watchdog_task.cancel() self.info("All results received, initiating shutdown") - else: - if not telemetry_ready_for_shutdown: - self.info("Waiting for telemetry results...") - if not server_metrics_ready_for_shutdown: - self.info("Waiting for server metrics results...") - if not accuracy_ready_for_shutdown: - self.info("Waiting for accuracy results...") + elif ( + pending_domains + := self._result_join_coordinator.pending_domains_changed() + ) is not None: + self.info(f"Waiting for result domains: {', '.join(pending_domains)}") # Call stop() OUTSIDE the lock to prevent deadlock if should_shutdown: @@ -1026,21 +967,19 @@ async def _cancel_profiling(self) -> None: ) ) self._profile_results = response.data - self._profile_results_received = True break except Exception as e: # Catch ANY exception during cancellation - we must always proceed to stop(). self.warning(f"Exception during cancel command (proceeding to stop): {e!r}") - # The normal completion path holds stop() on the accuracy shutdown gate + # The normal completion path holds stop() on the accuracy result domain # until RecordsManager publishes ProcessAccuracyResultMessage; the cancel - # path bypasses that gate, so wait a bounded time here for the graded + # path bypasses the barrier, so wait a bounded time here for the graded # summary to arrive before exporting (otherwise a cancelled accuracy run # can export with accuracy.* rows missing). if ( should_call_stop - and self._should_wait_for_accuracy - and self._accuracy_results is None + and "accuracy" in self._result_join_coordinator.pending_domains ): await self._await_accuracy_results_for_cancel() @@ -1050,24 +989,19 @@ async def _cancel_profiling(self) -> None: await asyncio.shield(self.stop()) async def _await_accuracy_results_for_cancel(self) -> None: - """Bounded poll for the accuracy summary on the cancel (Ctrl+C) path. - - Waits up to ``Environment.ACCURACY.CANCEL_RESULT_WAIT_SEC`` for the - RecordsManager's pub/sub summary message to land, returning as soon as it - does. Gates on ``_should_wait_for_accuracy`` (flipped false when the - message arrives) rather than ``_accuracy_results``, since a legitimately - empty run publishes ``results=None`` -- polling the results field would - never observe that arrival and would burn the full timeout. + """Bounded wait for the accuracy summary on the cancel (Ctrl+C) path. + + Waits up to ``Environment.ACCURACY.CANCEL_RESULT_WAIT_SEC`` for + RecordsManager's ``ProcessAccuracyResultMessage`` to land, returning as + soon as it does. Awaits ``_accuracy_result_arrived`` (set even on a + ``results=None`` empty run) so the wait ends on message arrival, not on + the results field. """ timeout = Environment.ACCURACY.CANCEL_RESULT_WAIT_SEC - interval = 0.05 - iterations = int(timeout / interval) - for _ in range(iterations): - if not self._should_wait_for_accuracy: - self.debug("Accuracy results arrived during cancel wait") - return - await asyncio.sleep(interval) - if self._should_wait_for_accuracy: + try: + await asyncio.wait_for(self._accuracy_result_arrived.wait(), timeout) + self.debug("Accuracy results arrived during cancel wait") + except TimeoutError: self.warning( "Accuracy results did not arrive within " f"{timeout}s of cancellation; export may omit accuracy metrics" diff --git a/src/aiperf/gpu_telemetry/manager.py b/src/aiperf/gpu_telemetry/manager.py index b0deac90bf..0d854a47b0 100644 --- a/src/aiperf/gpu_telemetry/manager.py +++ b/src/aiperf/gpu_telemetry/manager.py @@ -5,10 +5,14 @@ import asyncio from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from aiperf.common.base_component_service import BaseComponentService -from aiperf.common.enums import CommAddress, CommandType +from aiperf.common.enums import ( + CommAddress, + CommandType, + make_result_producer_capability, +) from aiperf.common.environment import Environment from aiperf.common.hooks import on_command, on_init, on_stop from aiperf.common.messages import ( @@ -56,6 +60,10 @@ class GPUTelemetryManager(BaseComponentService): service_id: Optional unique identifier for this service instance """ + extra_capabilities: ClassVar[tuple[str, ...]] = ( + make_result_producer_capability("telemetry"), + ) + def __init__( self, run: BenchmarkRun, diff --git a/src/aiperf/records/records_manager.py b/src/aiperf/records/records_manager.py index 7138a7bd71..cd9fdb6f83 100644 --- a/src/aiperf/records/records_manager.py +++ b/src/aiperf/records/records_manager.py @@ -22,6 +22,7 @@ CommandType, CreditPhase, MessageType, + make_result_producer_capability, ) from aiperf.common.environment import Environment from aiperf.common.hooks import background_task, on_command, on_message, on_pull_message @@ -379,6 +380,13 @@ def __init__( **kwargs, ) + # Advertise the result domains this manager produces so the controller's + # ResultJoinCoordinator waits for them on shutdown. Accuracy is a separate + # domain, joined only when accuracy mode is enabled for this run. + self.extra_capabilities = (make_result_producer_capability("profile"),) + if run.cfg.accuracy is not None and run.cfg.accuracy.enabled: + self.extra_capabilities += (make_result_producer_capability("accuracy"),) + self._records_tracker = RecordsTracker() self._error_tracker = ErrorTracker() diff --git a/src/aiperf/server_metrics/manager.py b/src/aiperf/server_metrics/manager.py index a6c6be655e..efbe53edff 100644 --- a/src/aiperf/server_metrics/manager.py +++ b/src/aiperf/server_metrics/manager.py @@ -4,10 +4,16 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar from aiperf.common.base_component_service import BaseComponentService -from aiperf.common.enums import CommAddress, CommandType, CreditPhase, MessageType +from aiperf.common.enums import ( + CommAddress, + CommandType, + CreditPhase, + MessageType, + make_result_producer_capability, +) from aiperf.common.environment import Environment from aiperf.common.hooks import on_command, on_message, on_stop from aiperf.common.messages import ( @@ -51,6 +57,10 @@ class ServerMetricsManager(BaseComponentService): service_id: Optional unique identifier for this service instance """ + extra_capabilities: ClassVar[tuple[str, ...]] = ( + make_result_producer_capability("server_metrics"), + ) + def __init__( self, run: BenchmarkRun, diff --git a/tests/unit/controller/test_accuracy_shutdown_gate.py b/tests/unit/controller/test_accuracy_shutdown_gate.py index 0aa8502adc..c4796b52ec 100644 --- a/tests/unit/controller/test_accuracy_shutdown_gate.py +++ b/tests/unit/controller/test_accuracy_shutdown_gate.py @@ -2,11 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for the SystemController accuracy shutdown-gate. -The controller sets ``_should_wait_for_accuracy`` from config at construction and -clears it only when a ``ProcessAccuracyResultMessage`` arrives. While the flag is -True and no accuracy summary has been received, ``_check_and_trigger_shutdown`` -must NOT trigger shutdown; once the message handler runs (with a real summary OR a -terminal ``results=None``), the flag clears and the accuracy term stops blocking. +Accuracy is a domain in the ``ResultJoinCoordinator`` shutdown barrier: RecordsManager +advertises a ``result_producer:accuracy`` capability (iff accuracy is enabled), which +registers the domain. While that domain is pending, ``_check_and_trigger_shutdown`` +must NOT trigger shutdown; once the ``ProcessAccuracyResultMessage`` handler runs +(with a real summary OR a terminal ``results=None``), the domain completes and the +barrier stops blocking. """ from __future__ import annotations @@ -73,17 +74,16 @@ def _summary() -> AccuracySummary: ) -class TestAccuracyShutdownGateEnabled: - """Accuracy ENABLED: the flag blocks shutdown until the message clears it.""" +def _register_records_manager(controller, *, accuracy: bool) -> None: + """Simulate RecordsManager registering its result-producer domains.""" + controller._result_join_coordinator.register("profile", "rm") + if accuracy: + controller._result_join_coordinator.register("accuracy", "rm") - @pytest.mark.asyncio - async def test_startup_sets_wait_flag_true( - self, benchmark_run, mock_service_manager - ) -> None: - controller = _build_controller( - benchmark_run, mock_service_manager, accuracy=True - ) - assert controller._should_wait_for_accuracy is True + +class TestAccuracyShutdownGateEnabled: + """Accuracy ENABLED: the accuracy domain blocks shutdown until its message + completes it.""" @pytest.mark.asyncio async def test_gate_blocks_shutdown_while_waiting( @@ -92,23 +92,25 @@ async def test_gate_blocks_shutdown_while_waiting( controller = _build_controller( benchmark_run, mock_service_manager, accuracy=True ) - # Profile results present so only the accuracy term can gate. - controller._profile_results_received = True - controller._accuracy_results = None + _register_records_manager(controller, accuracy=True) + # Profile complete so only the accuracy domain can gate. + controller._result_join_coordinator.complete_domain("profile") await controller._check_and_trigger_shutdown() + assert "accuracy" in controller._result_join_coordinator.pending_domains assert controller._shutdown_triggered is False controller.stop.assert_not_called() @pytest.mark.asyncio - async def test_summary_message_clears_flag_and_unblocks( + async def test_summary_message_completes_domain_and_unblocks( self, benchmark_run, mock_service_manager ) -> None: controller = _build_controller( benchmark_run, mock_service_manager, accuracy=True ) - controller._profile_results_received = True + _register_records_manager(controller, accuracy=True) + controller._result_join_coordinator.complete_domain("profile") summary = _summary() await controller._on_process_accuracy_result_message( @@ -118,21 +120,22 @@ async def test_summary_message_clears_flag_and_unblocks( ) ) - assert controller._should_wait_for_accuracy is False + assert controller._result_join_coordinator.ready is True assert controller._accuracy_results == summary assert controller._shutdown_triggered is True controller.stop.assert_awaited_once() @pytest.mark.asyncio - async def test_terminal_none_message_clears_flag_and_unblocks( + async def test_terminal_none_message_completes_domain_and_unblocks( self, benchmark_run, mock_service_manager ) -> None: - """A ``results=None`` terminal message must still clear the wait flag so a - summary that could not be computed does not hang shutdown forever.""" + """A ``results=None`` terminal message must still complete the accuracy + domain so a summary that could not be computed does not hang shutdown.""" controller = _build_controller( benchmark_run, mock_service_manager, accuracy=True ) - controller._profile_results_received = True + _register_records_manager(controller, accuracy=True) + controller._result_join_coordinator.complete_domain("profile") await controller._on_process_accuracy_result_message( ProcessAccuracyResultMessage( @@ -141,7 +144,7 @@ async def test_terminal_none_message_clears_flag_and_unblocks( ) ) - assert controller._should_wait_for_accuracy is False + assert controller._result_join_coordinator.ready is True assert controller._accuracy_results is None assert controller._shutdown_triggered is True controller.stop.assert_awaited_once() @@ -195,26 +198,29 @@ def test_no_injection_when_no_summary( class TestAccuracyShutdownGateDisabled: - """Accuracy DISABLED: the accuracy term never blocks shutdown.""" + """Accuracy DISABLED: RecordsManager advertises no accuracy domain, so it + never blocks shutdown.""" @pytest.mark.asyncio - async def test_startup_wait_flag_false( + async def test_accuracy_domain_never_registered( self, benchmark_run, mock_service_manager ) -> None: controller = _build_controller( benchmark_run, mock_service_manager, accuracy=False ) - assert controller._should_wait_for_accuracy is False + _register_records_manager(controller, accuracy=False) + + assert "accuracy" not in controller._result_join_coordinator.pending_domains @pytest.mark.asyncio - async def test_accuracy_term_never_blocks( + async def test_completing_profile_alone_unblocks( self, benchmark_run, mock_service_manager ) -> None: controller = _build_controller( benchmark_run, mock_service_manager, accuracy=False ) - controller._profile_results_received = True - controller._accuracy_results = None + _register_records_manager(controller, accuracy=False) + controller._result_join_coordinator.complete_domain("profile") await controller._check_and_trigger_shutdown() diff --git a/tests/unit/controller/test_result_join_coordinator.py b/tests/unit/controller/test_result_join_coordinator.py new file mode 100644 index 0000000000..f0098fcabd --- /dev/null +++ b/tests/unit/controller/test_result_join_coordinator.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from aiperf.controller.result_join_coordinator import ResultJoinCoordinator + + +def test_ready_when_no_result_producers_registered() -> None: + coord = ResultJoinCoordinator() + + assert coord.ready + assert coord.pending_domains == () + + +def test_register_marks_domain_pending_until_service_completes() -> None: + coord = ResultJoinCoordinator() + + coord.register("telemetry", "t1") + + assert not coord.ready + assert coord.pending_domains == ("telemetry",) + + coord.complete("telemetry", "t1") + + assert coord.ready + assert coord.pending_domains == () + + +def test_multiple_services_in_same_domain_all_must_complete() -> None: + coord = ResultJoinCoordinator() + + coord.register("telemetry", "t1") + coord.register("telemetry", "t2") + coord.complete("telemetry", "t1") + + assert not coord.ready + assert coord.pending_domains == ("telemetry",) + + coord.complete("telemetry", "t2") + + assert coord.ready + assert coord.pending_domains == () + + +def test_complete_domain_marks_all_participants_complete() -> None: + coord = ResultJoinCoordinator() + + coord.register("telemetry", "t1") + coord.register("telemetry", "t2") + coord.complete_domain("telemetry") + + assert coord.ready + assert coord.pending_domains == () + + +def test_unregister_removes_pending_participant() -> None: + coord = ResultJoinCoordinator() + + coord.register("telemetry", "t1") + coord.unregister("telemetry", "t1") + + assert coord.ready + assert coord.pending_domains == () + + +def test_unregister_service_removes_service_from_all_domains() -> None: + coord = ResultJoinCoordinator() + + coord.register("telemetry", "t1") + coord.register("server_metrics", "t1") + coord.register("profile", "records") + + coord.unregister_service("t1") + + assert coord.pending_domains == ("profile",) + + +def test_complete_unknown_participant_does_not_create_domain() -> None: + coord = ResultJoinCoordinator() + + coord.complete("telemetry", "t1") + + assert coord.ready + assert coord.pending_domains == () + + +def test_complete_unknown_domain_does_not_create_domain() -> None: + coord = ResultJoinCoordinator() + + coord.complete_domain("telemetry") + + assert coord.ready + assert coord.pending_domains == () + + +def test_completed_participant_reregistration_stays_complete() -> None: + coord = ResultJoinCoordinator() + + coord.register("telemetry", "t1") + coord.complete("telemetry", "t1") + coord.register("telemetry", "t1") + + assert coord.ready + assert coord.pending_domains == () + + +def test_pending_domains_changed_only_reports_changes() -> None: + coord = ResultJoinCoordinator() + + assert coord.pending_domains_changed() is None + + coord.register("server_metrics", "s1") + assert coord.pending_domains_changed() == ("server_metrics",) + assert coord.pending_domains_changed() is None + + coord.register("telemetry", "t1") + assert coord.pending_domains_changed() == ("server_metrics", "telemetry") + + coord.complete("telemetry", "t1") + assert coord.pending_domains_changed() == ("server_metrics",) + + coord.complete("server_metrics", "s1") + assert coord.pending_domains_changed() == () + assert coord.pending_domains_changed() is None diff --git a/tests/unit/controller/test_system_controller.py b/tests/unit/controller/test_system_controller.py index 6c49ae08f8..ecbbb0481a 100644 --- a/tests/unit/controller/test_system_controller.py +++ b/tests/unit/controller/test_system_controller.py @@ -703,66 +703,3 @@ async def test_extends_grace_when_api_process_alive_and_running( monkeypatch.setattr(Environment.API_SERVER, "POST_COMPLETE_GRACE", 7.0) sleeps = await self._drive_stop(system_controller, monkeypatch) assert sleeps[0] == 7.0 - - -class TestResultWatchdog: - """The side-channel result watchdog bounds the normal-completion shutdown so a - result message that is never published cannot hang the run.""" - - @staticmethod - def _controller() -> SystemController: - sc = SystemController.__new__(SystemController) - sc._shutdown_triggered = False - sc._should_wait_for_telemetry = False - sc._telemetry_results = None - sc._should_wait_for_server_metrics = False - sc._server_metrics_results = None - sc._should_wait_for_accuracy = False - sc._accuracy_results = None - sc.warning = MagicMock() - sc._check_and_trigger_shutdown = AsyncMock() - return sc - - @pytest.mark.asyncio - async def test_watchdog_releases_stalled_accuracy_gate(self) -> None: - sc = self._controller() - sc._should_wait_for_accuracy = True # message never arrived - - await sc._result_watchdog() - - assert sc._should_wait_for_accuracy is False - sc.warning.assert_called_once() - assert "accuracy" in sc.warning.call_args[0][0] - sc._check_and_trigger_shutdown.assert_awaited_once() - - @pytest.mark.asyncio - async def test_watchdog_noop_when_result_already_arrived(self) -> None: - sc = self._controller() - sc._should_wait_for_accuracy = True - sc._accuracy_results = object() # arrived before the timeout - - await sc._result_watchdog() - - sc.warning.assert_not_called() - sc._check_and_trigger_shutdown.assert_not_awaited() - - @pytest.mark.asyncio - async def test_watchdog_noop_when_shutdown_already_triggered(self) -> None: - sc = self._controller() - sc._should_wait_for_accuracy = True - sc._shutdown_triggered = True - - await sc._result_watchdog() - - sc.warning.assert_not_called() - sc._check_and_trigger_shutdown.assert_not_awaited() - - @pytest.mark.asyncio - async def test_arm_is_idempotent(self) -> None: - sc = self._controller() - sc._result_watchdog_task = None - sc._arm_result_watchdog() - first = sc._result_watchdog_task - sc._arm_result_watchdog() - assert sc._result_watchdog_task is first - first.cancel()