Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/benchmark-datasets.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ aiperf profile \
--fixed-schedule
```

For an endpoint such as SGLang that expects the session identifier in the request body, add `--session-header x-dynamo-session-id --session-body-field session_id`. This copies each recorded Exgentic session ID into the top-level `session_id` field without changing the dataset.

`source_model` selects the model that produced the trace; `--model` selects the target model receiving the replay. `benchmark` selects an Exgentic v2 workload. Invalid filters report the available harness/model combinations. The v1 dataset contains 22 combinations across five harnesses and six canonical source models.

Fixed-schedule mode emits each recorded call as an independently scheduled one-turn request using its start offset from the source session. Calls that overlapped in the trace therefore overlap during replay. Selected source sessions start together at offset zero. Without `--fixed-schedule`, each source session remains a closed-loop multi-turn conversation: AIPerf waits for one live response before applying the recorded residual delay and sending the next request.
Expand Down
8 changes: 8 additions & 0 deletions docs/cli-options.md
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,10 @@ Content type for request body serialization. By default, requests are sent as 'a

HTTP header name used to carry the per-session affinity identifier. When set, replaces the default `X-Correlation-ID` header with the provided name (e.g., `--session-header X-Session-ID`).

#### `--session-body-field` `<str>`

Optional request body field used to carry the same per-session affinity identifier as `--session-header` (or `X-Correlation-ID` by default). For example, use `--session-body-field session_id` for SGLang session-aware requests.

### Tokenizer

#### `--tokenizer` `<str>`
Expand Down Expand Up @@ -1786,6 +1790,10 @@ Content type for request body serialization. By default, requests are sent as 'a

HTTP header name used to carry the per-session affinity identifier. When set, replaces the default `X-Correlation-ID` header with the provided name (e.g., `--session-header X-Session-ID`).

#### `--session-body-field` `<str>`

Optional request body field used to carry the same per-session affinity identifier as `--session-header` (or `X-Correlation-ID` by default). For example, use `--session-body-field session_id` for SGLang session-aware requests.

### Tokenizer

#### `--tokenizer` `<str>`
Expand Down
6 changes: 6 additions & 0 deletions src/aiperf/common/models/model_endpoint_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,11 @@ def _redact_headers(self, value: list[tuple[str, str]]) -> list[tuple[str, str]]
description="HTTP header name to use for the per-session affinity identifier. "
"When set, replaces the default `X-Correlation-ID` header name with this value.",
)
session_body_field: str | None = Field(
default=None,
description="Optional request body field used to carry the same per-session "
"affinity identifier as the configured session header.",
)
collect_trace_chunks: bool = Field(
default=False,
description="Collect per-chunk trace data (timestamps and sizes) for HTTP trace export. "
Expand Down Expand Up @@ -213,6 +218,7 @@ def from_run(cls, run: BenchmarkRun) -> ModelEndpointInfo:
collect_trace_chunks=False,
template=getattr(ep, "template", None),
session_header=getattr(ep, "session_header", None),
session_body_field=getattr(ep, "session_body_field", None),
),
transport=ep.transport,
)
Expand Down
11 changes: 11 additions & 0 deletions src/aiperf/config/endpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,17 @@ def _redact_headers(self, value: dict[str, str]) -> dict[str, str]:
),
]

session_body_field: Annotated[
str | None,
Field(
default=None,
description=(
"Optional request body field used to carry the same per-session "
"affinity identifier as the configured session header."
),
),
]

wait_for_model_timeout: Annotated[
float,
Field(
Expand Down
1 change: 1 addition & 0 deletions src/aiperf/config/flags/_converter_endpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ def _endpoint_template_fallback(endpoint: dict[str, Any]) -> None:
"download_video_content": "download_video_content",
"request_content_type": "request_content_type",
"session_header": "session_header",
"session_body_field": "session_body_field",
}


Expand Down
1 change: 1 addition & 0 deletions src/aiperf/config/flags/_section_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"model_names",
"model_selection_strategy",
"request_content_type",
"session_body_field",
"session_header",
"streaming",
"timeout_seconds",
Expand Down
16 changes: 16 additions & 0 deletions src/aiperf/config/flags/cli_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,22 @@ class CLIConfig(BaseConfig):
),
] = None

session_body_field: Annotated[
str | None,
Field(
description=(
"Optional request body field used to carry the same per-session "
"affinity identifier as `--session-header` (or `X-Correlation-ID` "
"by default). For example, use `--session-body-field session_id` "
"for SGLang session-aware requests."
),
),
CLIParameter(
name=("--session-body-field",),
group=Groups.ENDPOINT,
),
] = None

@property
def url(self) -> str:
"""Return the first URL for backward compatibility."""
Expand Down
13 changes: 13 additions & 0 deletions src/aiperf/config/schema/aiperf-config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -7214,6 +7214,19 @@
"description": "HTTP header name used to carry the per-session affinity identifier. When set, replaces the default `X-Correlation-ID` header. Useful when the inference server expects a custom session-affinity header (e.g. `--session-header X-Session-ID`).",
"title": "Sessionheader"
},
"sessionBodyField": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Optional request body field used to carry the same per-session affinity identifier as the configured session header.",
"title": "Sessionbodyfield"
},
"waitForModelTimeout": {
"default": 0.0,
"description": "Enable a pre-flight readiness probe by setting this to a positive value (seconds). aiperf applies this timeout to each URL/model probe before starting the benchmark, aborting with a non-zero exit if any probe times out. For multiple URLs or models, worst-case wall-clock time can be roughly this timeout multiplied by the number of URL/model probes. The probe strategy is controlled by `--wait-for-model-mode`, which defaults to sending a 1-token inference request. 0 (default) disables the probe. Eliminates the need for external shell-based readiness loops in containers and Kubernetes recipes.",
Expand Down
7 changes: 7 additions & 0 deletions src/aiperf/transports/aiohttp_transport.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,13 @@ async def send_request(
try:
url = self.build_url(request_info)
headers = self.build_headers(request_info)
session_body_field = request_info.model_endpoint.endpoint.session_body_field
if session_body_field:
session_header = (
request_info.model_endpoint.endpoint.session_header
or "X-Correlation-ID"
)
payload[session_body_field] = headers[session_header]
Comment on lines +322 to +328

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win

Guard the header lookup to avoid silent per-request failures.

headers[session_header] will raise KeyError whenever the configured (or default X-Correlation-ID) header isn't present in the built headers for a given request/turn. Because send_request's broad except Exception swallows this, a misconfigured --session-header/--session-body-field pair — or a turn missing the expected header — silently fails every request with an opaque error instead of a clear diagnostic.

🐛 Proposed fix using a safe lookup
-            session_body_field = request_info.model_endpoint.endpoint.session_body_field
-            if session_body_field:
-                session_header = (
-                    request_info.model_endpoint.endpoint.session_header
-                    or "X-Correlation-ID"
-                )
-                payload[session_body_field] = headers[session_header]
+            session_body_field = request_info.model_endpoint.endpoint.session_body_field
+            if session_body_field:
+                session_header = (
+                    request_info.model_endpoint.endpoint.session_header
+                    or "X-Correlation-ID"
+                )
+                session_value = headers.get(session_header)
+                if session_value is not None:
+                    payload[session_body_field] = session_value
+                else:
+                    self.warning(
+                        f"session_body_field={session_body_field!r} configured but "
+                        f"header {session_header!r} not found in request headers; "
+                        "skipping body-field passthrough for this request."
+                    )
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
session_body_field = request_info.model_endpoint.endpoint.session_body_field
if session_body_field:
session_header = (
request_info.model_endpoint.endpoint.session_header
or "X-Correlation-ID"
)
payload[session_body_field] = headers[session_header]
session_body_field = request_info.model_endpoint.endpoint.session_body_field
if session_body_field:
session_header = (
request_info.model_endpoint.endpoint.session_header
or "X-Correlation-ID"
)
session_value = headers.get(session_header)
if session_value is not None:
payload[session_body_field] = session_value
else:
self.warning(
f"session_body_field={session_body_field!r} configured but "
f"header {session_header!r} not found in request headers; "
"skipping body-field passthrough for this request."
)
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/aiperf/transports/aiohttp_transport.py` around lines 322 - 328, The
session-body injection in aiohttp_transport.send_request currently does a direct
headers[session_header] lookup, which can raise KeyError and get swallowed by
the broad exception handling. Update the session_body_field block to use a safe
header lookup with an explicit fallback/diagnostic when
request_info.model_endpoint.endpoint.session_header (or default
X-Correlation-ID) is missing, and surface a clear error before payload
assignment. Keep the fix localized around send_request and the
session_header/session_body_field handling so misconfigured headers fail with a
useful message instead of silently breaking every request.

use_form_data = (
request_info.model_endpoint.endpoint.request_content_type
== RequestContentType.MULTIPART_FORM_DATA
Expand Down
17 changes: 17 additions & 0 deletions tests/unit/config/test_converter_endpoint_session.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from aiperf.config.flags._converter_endpoint import build_endpoint
from aiperf.config.flags.cli_config import CLIConfig


def test_session_affinity_fields_are_added_to_endpoint() -> None:
endpoint = build_endpoint(
CLIConfig(
session_header="x-dynamo-session-id",
session_body_field="session_id",
)
)

assert endpoint["session_header"] == "x-dynamo-session-id"
assert endpoint["session_body_field"] == "session_id"
22 changes: 22 additions & 0 deletions tests/unit/transports/test_aiohttp_transport.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@
from contextlib import asynccontextmanager
from unittest.mock import AsyncMock, MagicMock, patch

import orjson
import pytest

from aiperf.common.enums import ConnectionReuseStrategy, CreditPhase
from aiperf.common.models import Turn
from aiperf.common.models.record_models import RequestInfo, RequestRecord
from aiperf.plugin import plugins
from aiperf.plugin.enums import TransportType
Expand Down Expand Up @@ -297,6 +299,26 @@ async def test_send_request_serializes_payload_with_orjson(
assert b"messages" in json_bytes
assert b"gpt-4" in json_bytes

@pytest.mark.asyncio
async def test_send_request_copies_resolved_session_header_to_body(self):
model_endpoint = create_model_endpoint_info()
model_endpoint.endpoint.session_header = "x-dynamo-session-id"
model_endpoint.endpoint.session_body_field = "session_id"
transport = AioHttpTransport(model_endpoint=model_endpoint)
await self._setup_initialized_transport_with_mock(transport)

request_info = create_request_info(model_endpoint)
request_info.turns = [
Turn(extra_headers={"x-dynamo-session-id": "source-session-1"})
]
payload = {"model": "test-model", "messages": []}

await transport.send_request(request_info, payload)

args = self._extract_call_args(transport.aiohttp_client.post_request.call_args)
assert args["headers"]["x-dynamo-session-id"] == "source-session-1"
assert orjson.loads(args["json_bytes"])["session_id"] == "source-session-1"

@pytest.mark.asyncio
async def test_send_request_handles_exception(
self, transport, model_endpoint_non_streaming
Expand Down
Loading