From bea7faea9f64e77a3963d12746450cf187305167 Mon Sep 17 00:00:00 2001 From: qinxuye Date: Mon, 15 Jun 2026 21:09:49 +0800 Subject: [PATCH 1/5] feat: add random serving benchmark workload --- benchmark/README.md | 17 ++- benchmark/benchmark_runner.py | 32 +++-- benchmark/benchmark_serving.py | 122 ++++++++++++++-- benchmark/utils.py | 252 ++++++++++++++++++++++++++++++++- 4 files changed, 400 insertions(+), 23 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index a24fd292f9..cbe2ff8f8d 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -14,7 +14,7 @@ This tool will sample prompts from dataset, and run benchmark with serialized re ```bash python benchmark_latency.py --dataset /path/to/ShareGPT_V3_unfiltered_cleaned_split.json \ --tokenizer /path/to/tokenizer \ - --num-prompt 100 \ + --num-prompts 100 \ --model-uid ${model_uid} ``` @@ -26,7 +26,20 @@ This tool will sample prompts from dataset, and run benchmark with parallel requ python benchmark_serving.py --dataset /path/to/ShareGPT_V3_unfiltered_cleaned_split.json \ --tokenizer /path/to/tokenizer \ --model-uid ${model_uid} \ - --num-prompt 100 --concurrency 50 + --num-prompts 100 --concurrency 50 +``` + +It can also generate synthetic prompts without a dataset, similar to vLLM's +random benchmark mode: + +```bash +python benchmark_serving.py --dataset-name random \ + --tokenizer /path/to/tokenizer \ + --model-uid ${model_uid} \ + --input-len 1024 --output-len 128 \ + --random-range-ratio 0.2 \ + --num-prompts 1000 --concurrency 50 \ + --stream --ignore-eos ``` ## Benchmarking long context serving diff --git a/benchmark/benchmark_runner.py b/benchmark/benchmark_runner.py index dd4a6fb143..c28a453715 100644 --- a/benchmark/benchmark_runner.py +++ b/benchmark/benchmark_runner.py @@ -12,16 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import aiohttp import json +import logging import sys +import time import traceback import warnings -import logging from dataclasses import dataclass, field -import time from typing import List, Optional, Tuple +import aiohttp import numpy as np logging.basicConfig(level=logging.INFO) @@ -56,6 +56,7 @@ def __init__( stream: bool, api_key: Optional[str] = None, print_error: bool = False, + ignore_eos: bool = False, ): self.api_url = api_url self.model_uid = model_uid @@ -65,6 +66,7 @@ def __init__( self.stream = stream self.api_key = api_key self.print_error = print_error + self.ignore_eos = ignore_eos async def run(self): await self.warm_up() @@ -107,6 +109,8 @@ async def send_request(self, request: tuple, warming_up: bool = False): "stream": False, "messages": [{"role": "user", "content": prompt}], } + if self.ignore_eos: + pload["ignore_eos"] = True headers = {"User-Agent": "Benchmark Client"} if self.api_key: @@ -142,7 +146,9 @@ async def send_request(self, request: tuple, warming_up: bool = False): if not chunk_bytes: continue - chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data:") + chunk = remove_prefix( + chunk_bytes.decode("utf-8"), "data:" + ) if chunk == "[DONE]": latency = time.perf_counter() - st @@ -157,18 +163,24 @@ async def send_request(self, request: tuple, warming_up: bool = False): # Decoding phase else: - output.itl.append(timestamp - most_recent_timestamp) + output.itl.append( + timestamp - most_recent_timestamp + ) most_recent_timestamp = timestamp output.latency = latency output.success = True - output.completion_tokens = data["usage"]["completion_tokens"] + output.completion_tokens = data["usage"][ + "completion_tokens" + ] else: resp = await response.json() output.latency = time.perf_counter() - st output.success = True - output.completion_tokens = resp["usage"]["completion_tokens"] + output.completion_tokens = resp["usage"][ + "completion_tokens" + ] except Exception: output.success = False exc_info = sys.exc_info() @@ -368,7 +380,9 @@ def print_stats(self): logger.info("Errors encountered during benchmark:") for output in self.outputs: if not output.success: - print(f"Error for prompt with length {output.prompt_len}: {output.error}") + print( + f"Error for prompt with length {output.prompt_len}: {output.error}" + ) else: logger.info( "Errors were encountered during the benchmark. Run with --print-error to see detailed error messages." @@ -385,6 +399,7 @@ def __init__( concurrency: int, api_key: Optional[str] = None, print_error: bool = False, + ignore_eos: bool = False, ): super().__init__( api_url, @@ -393,6 +408,7 @@ def __init__( stream, api_key, print_error, + ignore_eos, ) self.concurrency = concurrency self.left = len(input_requests) diff --git a/benchmark/benchmark_serving.py b/benchmark/benchmark_serving.py index cc56750f27..3845e3eb21 100644 --- a/benchmark/benchmark_serving.py +++ b/benchmark/benchmark_serving.py @@ -14,20 +14,48 @@ import argparse import asyncio +import json import logging import random -from typing import List, Tuple, Optional +from typing import List, Optional, Tuple import numpy as np - -from utils import sample_requests, get_tokenizer from benchmark_runner import ConcurrentBenchmarkRunner - +from utils import get_tokenizer, sample_random_requests, sample_requests logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +def parse_random_range_ratio(value): + try: + return float(value) + except ValueError: + pass + + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + raise argparse.ArgumentTypeError( + "--random-range-ratio must be a float or a JSON object with " + "'input' and 'output' keys." + ) from exc + if not isinstance(parsed, dict): + raise argparse.ArgumentTypeError( + "--random-range-ratio JSON value must be an object." + ) + try: + return {"input": float(parsed["input"]), "output": float(parsed["output"])} + except KeyError as exc: + raise argparse.ArgumentTypeError( + "--random-range-ratio JSON object must contain 'input' and 'output'." + ) from exc + except (TypeError, ValueError) as exc: + raise argparse.ArgumentTypeError( + "--random-range-ratio input and output values must be numbers." + ) from exc + + class ServingBenchmarkRunner(ConcurrentBenchmarkRunner): def __init__( self, @@ -39,6 +67,7 @@ def __init__( request_rate: float, api_key: Optional[str] = None, print_error: bool = False, + ignore_eos: bool = False, ): super().__init__( api_url, @@ -48,6 +77,7 @@ def __init__( concurrency, api_key, print_error, + ignore_eos, ) self.request_rate = request_rate self.queue = None # delay the creation of the queue @@ -103,12 +133,27 @@ def main(args: argparse.Namespace): logger.info("Preparing for benchmark.") tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) - input_requests = sample_requests( - args.dataset, - args.num_prompts, - tokenizer, - prompt_len_limit=args.prompt_len_limit, - ) + if args.dataset_name == "sharegpt": + if not args.dataset: + raise ValueError("--dataset is required when --dataset-name=sharegpt.") + input_requests = sample_requests( + args.dataset, + args.num_prompts, + tokenizer, + prompt_len_limit=args.prompt_len_limit, + ) + elif args.dataset_name == "random": + input_requests = sample_random_requests( + args.num_prompts, + tokenizer, + input_len=args.input_len, + output_len=args.output_len, + range_ratio=args.random_range_ratio, + prefix_len=args.random_prefix_len, + seed=args.seed, + ) + else: + raise ValueError(f"Unsupported dataset name: {args.dataset_name}") logger.info("Benchmark starts.") @@ -121,6 +166,7 @@ def main(args: argparse.Namespace): concurrency=args.concurrency, api_key=args.api_key, print_error=args.print_error, + ignore_eos=args.ignore_eos, ) asyncio.run(benchmark.run()) @@ -134,7 +180,17 @@ def main(args: argparse.Namespace): parser.add_argument("--host", type=str, default="localhost") parser.add_argument("--port", type=int, default=9997) parser.add_argument( - "--dataset", type=str, required=True, help="Path to the dataset." + "--dataset-name", + type=str, + choices=["sharegpt", "random"], + default="sharegpt", + help="Dataset source to use for benchmark requests.", + ) + parser.add_argument( + "--dataset", + type=str, + default=None, + help="Path to the dataset. Required when --dataset-name=sharegpt.", ) parser.add_argument( "--tokenizer", type=str, required=True, help="Name or path of the tokenizer." @@ -145,6 +201,43 @@ def main(args: argparse.Namespace): parser.add_argument( "--prompt-len-limit", type=int, default=1024, help="Prompt length limitation." ) + parser.add_argument( + "--input-len", + type=int, + default=1024, + help="Input token length for random dataset requests.", + ) + parser.add_argument( + "--output-len", + type=int, + default=128, + help="Output token length for random dataset requests.", + ) + parser.add_argument( + "--random-input-len", + dest="input_len", + type=int, + help="Alias for --input-len.", + ) + parser.add_argument( + "--random-output-len", + dest="output_len", + type=int, + help="Alias for --output-len.", + ) + parser.add_argument( + "--random-range-ratio", + type=parse_random_range_ratio, + default=0.0, + help="Sample random input/output lengths from length * (1 +/- ratio). " + 'Can also be a JSON object like \'{"input": 0.3, "output": 0.5}\'.', + ) + parser.add_argument( + "--random-prefix-len", + type=int, + default=0, + help="Fixed prefix token length prepended to random dataset prompts.", + ) parser.add_argument( "--api-key", type=str, @@ -177,10 +270,15 @@ def main(args: argparse.Namespace): parser.add_argument( "--stream", action="store_true", help="Enable streaming responses." ) + parser.add_argument( + "--ignore-eos", + action="store_true", + help="Ask the backend to ignore EOS so random outputs approach max_tokens.", + ) parser.add_argument( "--print-error", action="store_true", - help="Print detailed error messages if any errors encountered." + help="Print detailed error messages if any errors encountered.", ) args = parser.parse_args() main(args) diff --git a/benchmark/utils.py b/benchmark/utils.py index 8372d2e916..2b44dd961f 100644 --- a/benchmark/utils.py +++ b/benchmark/utils.py @@ -14,9 +14,11 @@ import json import logging +import math import random -from typing import TYPE_CHECKING, List, Tuple +from typing import TYPE_CHECKING, Dict, List, Tuple, Union +import numpy as np from transformers import AutoTokenizer, PreTrainedTokenizerFast logging.basicConfig(level=logging.INFO) @@ -26,6 +28,8 @@ if TYPE_CHECKING: from transformers import PreTrainedTokenizerBase +RangeRatio = Union[float, Dict[str, float]] + # A fast LLaMA tokenizer with the pre-processed `tokenizer.json` file. _FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer" @@ -140,6 +144,252 @@ def sample_requests( return sampled_requests +def _resolve_range_ratios(range_ratio: RangeRatio) -> Tuple[float, float]: + if isinstance(range_ratio, dict): + try: + return float(range_ratio["input"]), float(range_ratio["output"]) + except KeyError as exc: + raise ValueError( + "When range_ratio is a dict it must contain 'input' and " + f"'output' keys, got: {sorted(range_ratio)}" + ) from exc + ratio = float(range_ratio) + return ratio, ratio + + +def _get_vocab_size(tokenizer: "PreTrainedTokenizerBase") -> int: + vocab_size = getattr(tokenizer, "vocab_size", None) + if not isinstance(vocab_size, int) or vocab_size <= 0: + vocab_size = len(tokenizer) + if vocab_size <= 0: + raise ValueError("Tokenizer vocab size must be positive.") + return vocab_size + + +def _num_special_tokens_to_add(tokenizer: "PreTrainedTokenizerBase") -> int: + num_special_tokens_to_add = getattr(tokenizer, "num_special_tokens_to_add", None) + if callable(num_special_tokens_to_add): + return int(num_special_tokens_to_add()) + return 0 + + +def _encode_prompt( + tokenizer: "PreTrainedTokenizerBase", + prompt: str, + add_special_tokens: bool = False, +) -> List[int]: + encode = getattr(tokenizer, "encode", None) + if callable(encode): + return list(encode(prompt, add_special_tokens=add_special_tokens)) + return list(tokenizer(prompt, add_special_tokens=add_special_tokens).input_ids) + + +def _get_sampling_params( + rng: np.random.Generator, + num_requests: int, + range_ratio: RangeRatio, + input_len: int, + output_len: int, + tokenizer: "PreTrainedTokenizerBase", +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + input_range_ratio, output_range_ratio = _resolve_range_ratios(range_ratio) + if not (0.0 <= input_range_ratio < 1.0): + raise ValueError("input_range_ratio must be in [0, 1).") + if not (0.0 <= output_range_ratio < 1.0): + raise ValueError("output_range_ratio must be in [0, 1).") + + num_special_tokens = _num_special_tokens_to_add(tokenizer) + real_input_len = max(0, int(input_len) - num_special_tokens) + input_low = math.floor(real_input_len * (1 - input_range_ratio)) + input_high = math.ceil(real_input_len * (1 + input_range_ratio)) + output_low = math.floor(output_len * (1 - output_range_ratio)) + output_high = math.ceil(output_len * (1 + output_range_ratio)) + output_low = max(output_low, 1) + output_high = max(output_high, 1) + + if input_low > input_high: + raise ValueError( + f"Invalid input sampling interval: low={input_low} > high={input_high}" + ) + if output_low > output_high: + raise ValueError( + f"Invalid output sampling interval: low={output_low} > high={output_high}" + ) + + logger.info( + "Sampling input_len from [%s, %s] and output_len from [%s, %s]", + input_low, + input_high, + output_low, + output_high, + ) + input_lens = rng.integers(input_low, input_high + 1, size=num_requests) + output_lens = rng.integers(output_low, output_high + 1, size=num_requests) + offsets = rng.integers(0, _get_vocab_size(tokenizer), size=num_requests) + return input_lens, output_lens, offsets + + +def _gen_prompt_decode_to_target_len( + tokenizer: "PreTrainedTokenizerBase", + token_sequence: List[int], + target_token_len: int, + rng: np.random.Generator, + max_retry: int = 10, +) -> Tuple[str, List[int], int]: + remain_num_try = max_retry + token_mismatch = 0 + vocab_size = _get_vocab_size(tokenizer) + while True: + prompt = tokenizer.decode(token_sequence) + token_sequence = _encode_prompt(tokenizer, prompt, add_special_tokens=False) + + if remain_num_try <= 0: + if len(token_sequence) != target_token_len: + token_mismatch = len(token_sequence) - target_token_len + break + if len(token_sequence) == target_token_len: + break + if len(token_sequence) < target_token_len: + extra_tokens = rng.integers( + 0, + vocab_size, + size=target_token_len - len(token_sequence), + ).tolist() + token_sequence.extend(extra_tokens) + else: + token_sequence = token_sequence[:target_token_len] + + remain_num_try -= 1 + return prompt, token_sequence, token_mismatch + + +def _get_random_prefix( + tokenizer: "PreTrainedTokenizerBase", + allowed_tokens: np.ndarray, + prefix_len: int, + rng: np.random.Generator, +) -> List[int]: + if prefix_len <= 0: + return [] + + prefix_tokens = allowed_tokens[ + rng.integers(0, len(allowed_tokens), size=prefix_len) + ].tolist() + _, adjusted_tokens, token_mismatch = _gen_prompt_decode_to_target_len( + tokenizer=tokenizer, + token_sequence=prefix_tokens, + target_token_len=prefix_len, + rng=rng, + ) + if token_mismatch != 0: + sign = "more" if token_mismatch > 0 else "fewer" + logger.warning( + "Prefix tokenization produced %d %s tokens than expected after " + "decoding and re-encoding.", + abs(token_mismatch), + sign, + ) + return adjusted_tokens + + +def _generate_prompt_with_target_len( + tokenizer: "PreTrainedTokenizerBase", + prefix_token_ids: List[int], + prefix_len: int, + input_len: int, + offset: int, + index: int, + allowed_tokens: np.ndarray, + rng: np.random.Generator, +) -> Tuple[str, int, int]: + inner_sequence = allowed_tokens[ + (offset + index + np.arange(input_len)) % len(allowed_tokens) + ].tolist() + token_sequence = prefix_token_ids + inner_sequence + total_input_len = prefix_len + int(input_len) + prompt, adjusted_token_sequence, token_mismatch = _gen_prompt_decode_to_target_len( + tokenizer=tokenizer, + token_sequence=token_sequence, + target_token_len=total_input_len, + rng=rng, + ) + return prompt, len(adjusted_token_sequence), token_mismatch + + +def sample_random_requests( + num_requests: int, + tokenizer: "PreTrainedTokenizerBase", + input_len: int = 1024, + output_len: int = 128, + range_ratio: RangeRatio = 0.0, + prefix_len: int = 0, + seed: int = 0, +) -> List[Tuple[str, int, int]]: + """Generate vLLM-style synthetic requests with reproducible token lengths.""" + if num_requests <= 0: + raise ValueError("num_requests must be positive.") + if prefix_len < 0: + raise ValueError("prefix_len must be non-negative.") + + rng = np.random.default_rng(seed) + + input_range_ratio, _ = _resolve_range_ratios(range_ratio) + num_special_tokens = _num_special_tokens_to_add(tokenizer) + real_input_len = max(0, int(input_len) - num_special_tokens) + min_sampled_input = math.floor(real_input_len * (1.0 - input_range_ratio)) + min_total_input = int(prefix_len) + min_sampled_input + if min_total_input < 1: + raise ValueError( + "--random-input-len is too small: with tokenizer special tokens " + f"{num_special_tokens} and input range ratio {input_range_ratio}, " + "the minimum possible total input tokens is " + f"{min_total_input}." + ) + + input_lens, output_lens, offsets = _get_sampling_params( + rng, + num_requests, + range_ratio, + input_len, + output_len, + tokenizer, + ) + vocab_size = _get_vocab_size(tokenizer) + prohibited_tokens = getattr(tokenizer, "all_special_ids", []) or [] + all_tokens = np.arange(vocab_size) + allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens))) + if len(allowed_tokens) == 0: + raise ValueError("Tokenizer has no non-special tokens for random sampling.") + + prefix_token_ids = _get_random_prefix(tokenizer, allowed_tokens, prefix_len, rng) + + sampled_requests = [] + token_mismatch_total = 0 + for i in range(num_requests): + prompt, prompt_len, token_mismatch = _generate_prompt_with_target_len( + tokenizer=tokenizer, + prefix_token_ids=prefix_token_ids, + prefix_len=prefix_len, + input_len=int(input_lens[i]), + offset=int(offsets[i]), + index=i, + allowed_tokens=allowed_tokens, + rng=rng, + ) + token_mismatch_total += token_mismatch + sampled_requests.append((prompt, prompt_len, int(output_lens[i]))) + + if token_mismatch_total != 0: + sign = "more" if token_mismatch_total > 0 else "fewer" + logger.warning( + "Across all generated prompts, there were %d %s tokens than " + "expected after decoding and re-encoding.", + abs(token_mismatch_total), + sign, + ) + return sampled_requests + + def generate_sorting_prompts( num_prompts: int, context_length: int, From 163b31f6e11b900886158ad0f1d211f8c490d0f7 Mon Sep 17 00:00:00 2001 From: qinxuye Date: Mon, 15 Jun 2026 21:20:39 +0800 Subject: [PATCH 2/5] fix: address random benchmark review feedback --- benchmark/benchmark_runner.py | 19 +++++++++++++------ benchmark/utils.py | 20 ++++++++++++++------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/benchmark/benchmark_runner.py b/benchmark/benchmark_runner.py index c28a453715..68f3d539c4 100644 --- a/benchmark/benchmark_runner.py +++ b/benchmark/benchmark_runner.py @@ -128,6 +128,7 @@ async def send_request(self, request: tuple, warming_up: bool = False): ) as response: if response.status == 200: if self.stream: + usage = None async for chunk_bytes in response.content: # { # "id": "chataec79465-dfea-46af-81b9-c28124063fc0", @@ -155,6 +156,7 @@ async def send_request(self, request: tuple, warming_up: bool = False): else: timestamp = time.perf_counter() data = json.loads(chunk) + usage = data.get("usage") or usage # First token if ttft == 0.0: @@ -171,16 +173,21 @@ async def send_request(self, request: tuple, warming_up: bool = False): output.latency = latency output.success = True - output.completion_tokens = data["usage"][ - "completion_tokens" - ] + output.completion_tokens = ( + usage["completion_tokens"] + if usage and "completion_tokens" in usage + else len(output.itl) + 1 + ) else: resp = await response.json() output.latency = time.perf_counter() - st output.success = True - output.completion_tokens = resp["usage"][ - "completion_tokens" - ] + usage = resp.get("usage") + output.completion_tokens = ( + usage["completion_tokens"] + if usage and "completion_tokens" in usage + else 0 + ) except Exception: output.success = False exc_info = sys.exc_info() diff --git a/benchmark/utils.py b/benchmark/utils.py index 2b44dd961f..aed4657831 100644 --- a/benchmark/utils.py +++ b/benchmark/utils.py @@ -233,12 +233,12 @@ def _gen_prompt_decode_to_target_len( tokenizer: "PreTrainedTokenizerBase", token_sequence: List[int], target_token_len: int, + allowed_tokens: np.ndarray, rng: np.random.Generator, max_retry: int = 10, ) -> Tuple[str, List[int], int]: remain_num_try = max_retry token_mismatch = 0 - vocab_size = _get_vocab_size(tokenizer) while True: prompt = tokenizer.decode(token_sequence) token_sequence = _encode_prompt(tokenizer, prompt, add_special_tokens=False) @@ -250,11 +250,13 @@ def _gen_prompt_decode_to_target_len( if len(token_sequence) == target_token_len: break if len(token_sequence) < target_token_len: - extra_tokens = rng.integers( - 0, - vocab_size, - size=target_token_len - len(token_sequence), - ).tolist() + extra_tokens = allowed_tokens[ + rng.integers( + 0, + len(allowed_tokens), + size=target_token_len - len(token_sequence), + ) + ].tolist() token_sequence.extend(extra_tokens) else: token_sequence = token_sequence[:target_token_len] @@ -279,6 +281,7 @@ def _get_random_prefix( tokenizer=tokenizer, token_sequence=prefix_tokens, target_token_len=prefix_len, + allowed_tokens=allowed_tokens, rng=rng, ) if token_mismatch != 0: @@ -311,6 +314,7 @@ def _generate_prompt_with_target_len( tokenizer=tokenizer, token_sequence=token_sequence, target_token_len=total_input_len, + allowed_tokens=allowed_tokens, rng=rng, ) return prompt, len(adjusted_token_sequence), token_mismatch @@ -328,6 +332,10 @@ def sample_random_requests( """Generate vLLM-style synthetic requests with reproducible token lengths.""" if num_requests <= 0: raise ValueError("num_requests must be positive.") + if input_len <= 0: + raise ValueError("input_len must be positive.") + if output_len <= 0: + raise ValueError("output_len must be positive.") if prefix_len < 0: raise ValueError("prefix_len must be non-negative.") From 3a261295219d78f578bb2e076adc6dc2f52cc25f Mon Sep 17 00:00:00 2001 From: qinxuye Date: Wed, 17 Jun 2026 19:30:22 +0800 Subject: [PATCH 3/5] fix: harden benchmark request validation --- benchmark/benchmark_runner.py | 6 ++++-- benchmark/benchmark_serving.py | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/benchmark/benchmark_runner.py b/benchmark/benchmark_runner.py index 68f3d539c4..33343f5cb7 100644 --- a/benchmark/benchmark_runner.py +++ b/benchmark/benchmark_runner.py @@ -175,7 +175,8 @@ async def send_request(self, request: tuple, warming_up: bool = False): output.success = True output.completion_tokens = ( usage["completion_tokens"] - if usage and "completion_tokens" in usage + if isinstance(usage, dict) + and isinstance(usage.get("completion_tokens"), int) else len(output.itl) + 1 ) else: @@ -185,7 +186,8 @@ async def send_request(self, request: tuple, warming_up: bool = False): usage = resp.get("usage") output.completion_tokens = ( usage["completion_tokens"] - if usage and "completion_tokens" in usage + if isinstance(usage, dict) + and isinstance(usage.get("completion_tokens"), int) else 0 ) except Exception: diff --git a/benchmark/benchmark_serving.py b/benchmark/benchmark_serving.py index 3845e3eb21..91928fec24 100644 --- a/benchmark/benchmark_serving.py +++ b/benchmark/benchmark_serving.py @@ -133,6 +133,9 @@ def main(args: argparse.Namespace): logger.info("Preparing for benchmark.") tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + if args.request_rate <= 0.0: + raise ValueError("--request-rate must be positive.") + if args.dataset_name == "sharegpt": if not args.dataset: raise ValueError("--dataset is required when --dataset-name=sharegpt.") From 0c4adb336ae09b7d261614063066d0660f0f922b Mon Sep 17 00:00:00 2001 From: qinxuye Date: Wed, 17 Jun 2026 22:42:34 +0800 Subject: [PATCH 4/5] fix: address benchmark review feedback --- benchmark/benchmark_serving.py | 2 +- benchmark/utils.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmark/benchmark_serving.py b/benchmark/benchmark_serving.py index 91928fec24..6f6d1be226 100644 --- a/benchmark/benchmark_serving.py +++ b/benchmark/benchmark_serving.py @@ -132,10 +132,10 @@ def main(args: argparse.Namespace): model_uid = args.model_uid logger.info("Preparing for benchmark.") - tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) if args.request_rate <= 0.0: raise ValueError("--request-rate must be positive.") + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) if args.dataset_name == "sharegpt": if not args.dataset: raise ValueError("--dataset is required when --dataset-name=sharegpt.") diff --git a/benchmark/utils.py b/benchmark/utils.py index aed4657831..8d8d668110 100644 --- a/benchmark/utils.py +++ b/benchmark/utils.py @@ -170,6 +170,8 @@ def _num_special_tokens_to_add(tokenizer: "PreTrainedTokenizerBase") -> int: num_special_tokens_to_add = getattr(tokenizer, "num_special_tokens_to_add", None) if callable(num_special_tokens_to_add): return int(num_special_tokens_to_add()) + if isinstance(num_special_tokens_to_add, int): + return num_special_tokens_to_add return 0 From 0c40f620294d51776207bb66c4a778008765c75a Mon Sep 17 00:00:00 2001 From: qinxuye Date: Thu, 18 Jun 2026 20:15:09 +0800 Subject: [PATCH 5/5] fix: address benchmark review follow-up --- benchmark/benchmark_runner.py | 17 +++++++++++------ benchmark/benchmark_serving.py | 17 +++++++++++++++-- benchmark/utils.py | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/benchmark/benchmark_runner.py b/benchmark/benchmark_runner.py index 33343f5cb7..83e75fee98 100644 --- a/benchmark/benchmark_runner.py +++ b/benchmark/benchmark_runner.py @@ -184,12 +184,17 @@ async def send_request(self, request: tuple, warming_up: bool = False): output.latency = time.perf_counter() - st output.success = True usage = resp.get("usage") - output.completion_tokens = ( - usage["completion_tokens"] - if isinstance(usage, dict) - and isinstance(usage.get("completion_tokens"), int) - else 0 - ) + if isinstance(usage, dict) and isinstance( + usage.get("completion_tokens"), int + ): + output.completion_tokens = usage["completion_tokens"] + else: + logger.warning( + "usage missing or malformed in non-streaming " + "response; completion_tokens set to 0 and " + "throughput stats will be inaccurate." + ) + output.completion_tokens = 0 except Exception: output.success = False exc_info = sys.exc_info() diff --git a/benchmark/benchmark_serving.py b/benchmark/benchmark_serving.py index 6f6d1be226..5710a7fedf 100644 --- a/benchmark/benchmark_serving.py +++ b/benchmark/benchmark_serving.py @@ -27,9 +27,15 @@ logger = logging.getLogger(__name__) +def _validate_random_range_ratio(value: float, name: str) -> float: + if not (0.0 <= value < 1.0): + raise argparse.ArgumentTypeError(f"{name} must be in [0, 1), got {value}.") + return value + + def parse_random_range_ratio(value): try: - return float(value) + return _validate_random_range_ratio(float(value), name="--random-range-ratio") except ValueError: pass @@ -45,7 +51,14 @@ def parse_random_range_ratio(value): "--random-range-ratio JSON value must be an object." ) try: - return {"input": float(parsed["input"]), "output": float(parsed["output"])} + return { + "input": _validate_random_range_ratio( + float(parsed["input"]), name="--random-range-ratio input" + ), + "output": _validate_random_range_ratio( + float(parsed["output"]), name="--random-range-ratio output" + ), + } except KeyError as exc: raise argparse.ArgumentTypeError( "--random-range-ratio JSON object must contain 'input' and 'output'." diff --git a/benchmark/utils.py b/benchmark/utils.py index 8d8d668110..9423cf53e9 100644 --- a/benchmark/utils.py +++ b/benchmark/utils.py @@ -367,7 +367,7 @@ def sample_random_requests( vocab_size = _get_vocab_size(tokenizer) prohibited_tokens = getattr(tokenizer, "all_special_ids", []) or [] all_tokens = np.arange(vocab_size) - allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens))) + allowed_tokens = np.setdiff1d(all_tokens, prohibited_tokens) if len(allowed_tokens) == 0: raise ValueError("Tokenizer has no non-special tokens for random sampling.")