[PENDING: #744, #712] fix: record a clean but short completion under ignore_eos as a failure

docs/cli_flags.md

@@ -391,7 +391,7 @@ Any extra keys in the dict are passed as kwargs to datasets.load_dataset(). |
 | `--server.type` | Enum (vllm, sglang, tgi, mock) | Type of model server being benchmarked. |
 | `--server.model_name` | str | Model name sent in each request. Auto-detected from the server if unset. |
 | `--server.base_url` | str | Base URL of the model server, e.g. 'http://localhost:8000'. |
-| `--server.ignore_eos` | boolean | Ask the server to keep generating past the end-of-sequence token so outputs hit the requested length. |
+| `--server.ignore_eos` | boolean | Ask the server to keep generating past the end-of-sequence token so outputs hit the requested length. A completion that then delivers fewer output tokens than its max_tokens is recorded as a failure (TruncatedResponseError). Set false for servers that ignore this field. |
 | `--server.api_key` | str | API key sent as a bearer token with each request. |
 | `--server.cert_path` | str | Path to a client TLS certificate file. |
 | `--server.key_path` | str | Path to the private key for the client TLS certificate. |

docs/reports.md

@@ -34,7 +34,12 @@ Here is an example snippet from a `summary_lifecycle_metrics.json` report:
     "throughput": {
       "requests_per_sec": 1.02,
       "total_tokens_per_sec": 676.12
-    }
+    },
+    "finish_reasons": {
+      "length": 478,
+      "stop": 2
+    },
+    "output_shortfalls": 2
   },
   "failures": {
     "count": 3,
@@ -85,8 +90,10 @@ Here is an example snippet from a `summary_lifecycle_metrics.json` report:
 ### Key Sections
 
 - **`load_summary`**: Details about the requested vs achieved load.
-- **`successes`**: Metrics for successful requests.
-- **`failures`**: Metrics for failed requests, including the per-label error breakdown.
+- **`successes`**: Metrics for successful requests. Two fields describe whether those requests ran to the length they asked for:
+  - `finish_reasons`: how many successful requests ended with each reason the server reported, verbatim (OpenAI `finish_reason`: `stop`, `length`, `tool_calls`, ...; Anthropic `stop_reason`: `end_turn`, `max_tokens`, ...). `length` and `max_tokens` mean the requested budget was delivered; anything else means the server halted on its own. Requests whose server reported no reason are not counted.
+  - `output_shortfalls`: how many successful requests delivered fewer output tokens than their `max_tokens` asked for. Delivered means the server's own `usage.completion_tokens` when it reported one, otherwise the client-side count. Without `ignore_eos` a shortfall is usually the model emitting EOS as intended, which is why it is an observation here rather than a failure.
+- **`failures`**: Metrics for failed requests, including the per-label error breakdown. A request fails on a non-200 status, on a transport error, or on a 200 whose body is not a completion: a body carrying a top-level `error` object (label `inbanderror`, `error_msg` is that object) or one with neither completion content nor `usage` (label `emptyresponseerror`). With `server.ignore_eos: true` (the default) a completion that delivered fewer output tokens than its `max_tokens` is also a failure (label `truncatedresponseerror`, `error_msg` names delivered-of-requested and the `finish_reason`): the server was asked to generate the full length and did not, whether it stopped early or capped the request. A server that ignores the `ignore_eos` field will report every natural stop this way; set `ignore_eos: false` for it. The body is kept as `response` in the per-request report in every case, and each per-request entry records the request's `max_tokens` alongside `info.response_metrics.finish_reason`.
 - **`goodput_metrics`**: (Optional) Goodput statistics if constraints were configured.
 
 ### Token Counts
@@ -98,4 +105,4 @@ with client tokenization as the fallback, and `token_count_mismatches` counts th
 where the two output counts disagree. Alongside it, `client_fallback_requests` says how many
 requests had no server number at all, per side. See
 [Token Accounting and Provenance](./metrics.md#token-accounting-and-provenance) for what each
-field is derived from and which one normalizes per-token latency.
+field is derived from and which one normalizes per-token latency.

e2e/tests/test_golden_accuracy_sim.py

@@ -155,7 +155,13 @@ async def test_golden_accuracy(tokenizer_key: str, api_type: str, streaming: boo
                     "type": "vllm",
                     "model_name": "golden-model",
                     "base_url": f"http://{sim.host}:{sim.port}",
-                    "ignore_eos": True,
+                    # The golden sim serves each case's n_tokens regardless of the
+                    # request's max_tokens (the mock datagen sends the client
+                    # default, 30). Under ignore_eos the 16- and 24-token cases
+                    # would rightly be classed as truncated (#655); this fixture
+                    # measures tokenization fidelity, not length compliance, so it
+                    # does not claim ignore_eos.
+                    "ignore_eos": False,
                 },
                 "tokenizer": {"pretrained_model_name_or_path": tokenizer_path},
                 "report": {

e2e/tests/test_metrics_fidelity.py

@@ -158,6 +158,11 @@ async def test_streaming_metrics_match_simulated_ground_truth(ttft_sec: float, i
     assert output_tokens["total"] == num_requests * output_tokens_per_request
     distribution_stats = {k: v for k, v in output_tokens.items() if k != "total"}
     assert set(distribution_stats.values()) == {float(output_tokens_per_request)}, distribution_stats
+    # Same ground truth seen through the #655 fields: every request ran to its
+    # budget, so the sim reports finish_reason "length" on all of them and no
+    # request fell short of its max_tokens.
+    assert successes["finish_reasons"] == {"length": num_requests}
+    assert successes["output_shortfalls"] == 0
     # Input-side usage is plumbed through too. Its exact value is the sim's own
     # tokenization of the random prompts, which this test cannot predict.
     assert successes["prompt_tokens"]["total"] > 0

e2e/tests/test_output_length_sim.py

@@ -0,0 +1,131 @@
+# Copyright 2026 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+End-to-end check of the requested-versus-delivered output length fields (#655)
+against llm-d-inference-sim, extending the ground-truth pattern of #614.
+
+The sim honours ``ignore_eos``: with it on, every response runs to exactly
+``max_tokens`` and reports ``finish_reason: length``; with it off, it stops at
+a random earlier point and reports ``finish_reason: stop``. That gives two
+known outcomes for the same config:
+
+- ``ignore_eos: true``: no request is short, so ``finish_reasons`` is all
+  ``length``, ``output_shortfalls`` is 0 and nothing is reclassified as failed.
+- ``ignore_eos: false``: short responses are legitimate. They stay successes,
+  ``finish_reasons`` shows the ``stop`` bucket, and ``output_shortfalls`` equals
+  the number of requests whose server-reported ``completion_tokens`` fell
+  below the ``max_tokens`` recorded on their per-request entry, which is the
+  cross-check between the two report files.
+
+The truncation *failure* (short under ``ignore_eos``) is a condition the sim
+does not produce on its own, so it lives in the Integration tier against a
+fake (tests/required/integration/test_truncated_response.py) per the #606
+"fake the conditions, never the oracle" rule.
+"""
+
+import pytest
+from utils.llm_d_inference_sim import LLMDInferenceSimRunner
+from utils.benchmark import run_benchmark_minimal
+from utils.net import get_free_port
+from utils.testdata import extract_tarball
+
+TEST_MODEL_NAME = "google/gemma-3-270m"
+TEST_MODEL_TARBALL = "e2e/testdata/models/google_gemma-3-270m.tar.gz"
+MAX_TOKENS = 40
+
+
+# Pins every request's max_tokens to `tokens` through the random datagen's
+# output distribution (a degenerate distribution: min == max == mean).
+def _exact_length_distribution(tokens: int) -> dict:
+    return {"min": tokens, "max": tokens, "mean": tokens, "std_dev": 0, "total_count": 100}
+
+
+# Runs 10 streaming completion requests (rate 2 for 5s) against the sim with the
+# given ignore_eos and max_tokens 40, and returns the summary and per-request
+# reports. Fails if the run itself failed or produced no reports.
+async def _run(ignore_eos: bool) -> tuple[dict, list]:
+    model_path = extract_tarball(TEST_MODEL_TARBALL)
+    async with LLMDInferenceSimRunner(
+        TEST_MODEL_NAME,
+        *("--time-to-first-token", "20"),
+        *("--inter-token-latency", "5"),
+        *("--max-num-seqs", "64"),
+        *("--seed", "42"),
+        port=get_free_port(),
+    ) as sim:
+        result = await run_benchmark_minimal(
+            {
+                "api": {"type": "completion", "streaming": True},
+                "data": {
+                    "type": "random",
+                    "input_distribution": _exact_length_distribution(8),
+                    "output_distribution": _exact_length_distribution(MAX_TOKENS),
+                },
+                "load": {"type": "constant", "stages": [{"rate": 2, "duration": 5}], "num_workers": 2},
+                "server": {
+                    "type": "vllm",
+                    "model_name": TEST_MODEL_NAME,
+                    "base_url": f"http://{sim.host}:{sim.port}",
+                    "ignore_eos": ignore_eos,
+                },
+                "tokenizer": {"pretrained_model_name_or_path": str(model_path)},
+                "report": {"request_lifecycle": {"summary": True, "per_stage": False, "per_request": True}},
+            }
+        )
+    assert result.success, "Benchmark failed"
+    assert result.reports, "No reports generated from benchmark"
+    return result.reports["summary_lifecycle_metrics.json"], result.reports["per_request_lifecycle_metrics.json"]
+
+
+# ignore_eos on, max_tokens 40 on all 10 requests: the sim delivers exactly 40
+# with finish_reason "length" every time, so failures == 0, successes == 10,
+# finish_reasons == {"length": 10}, output_shortfalls == 0, and every per-request
+# entry records max_tokens 40 with completion_tokens 40.
+@pytest.mark.asyncio
+@pytest.mark.skipif(not LLMDInferenceSimRunner.is_available(), reason="local environment missing llm-d-inference-sim")
+async def test_full_budget_under_ignore_eos_reports_length_and_no_shortfall():
+    summary, per_request = await _run(ignore_eos=True)
+
+    assert summary["failures"]["count"] == 0
+    assert summary["successes"]["count"] == len(per_request) == 10
+    assert summary["successes"]["finish_reasons"] == {"length": 10}
+    assert summary["successes"]["output_shortfalls"] == 0
+    for entry in per_request:
+        assert entry["max_tokens"] == MAX_TOKENS
+        assert entry["info"]["response_metrics"]["server_usage"]["completion_tokens"] == MAX_TOKENS
+        assert entry["info"]["response_metrics"]["finish_reason"] == "length"
+
+
+# ignore_eos off, same config: the sim stops early at random, so short responses
+# are legitimate and stay successes (failures == 0). Every request carries a
+# finish_reason (buckets sum to 10, keys within {"stop", "length"}), and
+# output_shortfalls equals the number of per-request entries whose server
+# completion_tokens is below their recorded max_tokens (40), which with 10
+# random-length responses is at least 1.
+@pytest.mark.asyncio
+@pytest.mark.skipif(not LLMDInferenceSimRunner.is_available(), reason="local environment missing llm-d-inference-sim")
+async def test_early_stops_without_ignore_eos_stay_successes_and_are_counted():
+    summary, per_request = await _run(ignore_eos=False)
+
+    assert summary["failures"]["count"] == 0
+    assert summary["successes"]["count"] == len(per_request) == 10
+    finish_reasons = summary["successes"]["finish_reasons"]
+    assert set(finish_reasons) <= {"stop", "length"}, finish_reasons
+    assert sum(finish_reasons.values()) == 10
+
+    short = [e for e in per_request if e["info"]["response_metrics"]["server_usage"]["completion_tokens"] < e["max_tokens"]]
+    assert all(e["max_tokens"] == MAX_TOKENS for e in per_request)
+    assert len(short) >= 1, "the sim produced no early stop in 10 requests; the ignore_eos=false path is untested"
+    assert summary["successes"]["output_shortfalls"] == len(short)

inference_perf/apis/anthropic_messages.py

@@ -264,13 +264,21 @@ def output_message(output_text: str) -> dict[str, Any]:
 
 async def parse_anthropic_stream_response(
     response: ClientResponse,
-) -> tuple[str, dict[str, Any], list[float], str, list[str], dict[str, Any] | None]:
+) -> tuple[str, dict[str, Any], list[float], str, list[str], dict[str, Any] | None, str | None]:
     extract_content, build_output_message = _build_anthropic_stream_handlers()
-    output_text, chunk_times, raw_content, response_chunks, server_usage = await parse_sse_stream(
+    output_text, chunk_times, raw_content, response_chunks, server_usage, stop_reason = await parse_sse_stream(
         response,
         extract_content=extract_content,
     )
-    return output_text, build_output_message(output_text), chunk_times, raw_content, response_chunks, server_usage
+    return (
+        output_text,
+        build_output_message(output_text),
+        chunk_times,
+        raw_content,
+        response_chunks,
+        server_usage,
+        stop_reason,
+    )
 
 
 class AnthropicMessagesAPIData(InferenceAPIData):
@@ -314,6 +322,7 @@ async def process_response(
                 raw_content,
                 response_chunks,
                 server_usage,
+                stop_reason,
             ) = await parse_anthropic_stream_response(response)
             input_tokens = (server_usage or {}).get("input_tokens")
             output_tokens = (server_usage or {}).get("output_tokens")
@@ -330,6 +339,7 @@ async def process_response(
                     output_tokens=output_len,
                     output_token_times=chunk_times,
                     server_usage=server_usage,
+                    finish_reason=stop_reason,
                 ),
                 lora_adapter=lora_adapter,
                 extra_info={"raw_response": raw_content, "output_message": output_message, "output_text": output_text},
@@ -350,7 +360,7 @@ async def process_response(
             request_metrics=RequestMetrics(
                 text=Text(input_tokens=int(input_tokens) if input_tokens is not None else self._count_prompt_tokens(tokenizer))
             ),
-            response_metrics=UnaryResponseMetrics(output_tokens=output_len),
+            response_metrics=UnaryResponseMetrics(output_tokens=output_len, finish_reason=data.get("stop_reason")),
             lora_adapter=lora_adapter,
             extra_info=extra_info,
         )

inference_perf/apis/base.py

@@ -26,6 +26,24 @@ class ResponseMetrics(BaseModel):
     # Last-seen `usage` dict reported by the server (streaming: the trailing
     # usage chunk; non-streaming: the response body's `usage`).
     server_usage: Optional[dict[str, Any]] = None
+    # Why the server stopped generating, verbatim as it reported it: OpenAI's
+    # `finish_reason` (`stop`, `length`, `tool_calls`, ...) or Anthropic's
+    # `stop_reason` (`end_turn`, `max_tokens`, ...). `length`/`max_tokens` mean
+    # the requested budget was delivered; anything else means the server
+    # halted on its own. None when the server did not report one.
+    finish_reason: Optional[str] = None
+
+    def delivered_output_tokens(self) -> int:
+        """Output tokens the server actually produced.
+
+        The server's own ``usage.completion_tokens`` when it reported one, since
+        that is an exact count; otherwise the client-side re-tokenization in
+        ``output_tokens``, which is an approximation (#564).
+        """
+        completion_tokens = self.server_usage.get("completion_tokens") if self.server_usage else None
+        if isinstance(completion_tokens, (int, float)) and not isinstance(completion_tokens, bool):
+            return int(completion_tokens)
+        return self.output_tokens
 
 
 class UnaryResponseMetrics(ResponseMetrics):
@@ -72,6 +90,10 @@ class RequestLifecycleMetric(BaseModel):
     response_data: Optional[str] = None
     info: InferenceInfo
     error: Optional[ErrorResponseInfo]
+    # The `max_tokens` the request body asked for, so requested-versus-delivered
+    # output length is computable at report time. None when the body carried no
+    # `max_tokens` (a replay whose trace omits it, or a client that never set one).
+    max_tokens: Optional[int] = None
 
     ttft_slo_sec: Optional[float] = None
     tpot_slo_sec: Optional[float] = None