From c818cc01815f0323837de92c90f97ecb05a07254 Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Thu, 14 May 2026 18:04:43 -0500 Subject: [PATCH] added reasoning and cached metrics coming from api response --- benchmarking/src/llmperf/common_metrics.py | 2 ++ benchmarking/src/llmperf/sambanova_client.py | 7 +++++++ benchmarking/src/performance_evaluation.py | 2 ++ benchmarking/src/vllm_benchmark.py | 4 ++++ 4 files changed, 15 insertions(+) diff --git a/benchmarking/src/llmperf/common_metrics.py b/benchmarking/src/llmperf/common_metrics.py index fe27c8646..b679ea200 100644 --- a/benchmarking/src/llmperf/common_metrics.py +++ b/benchmarking/src/llmperf/common_metrics.py @@ -38,3 +38,5 @@ NUM_OUTPUT_TOKENS_SERVER = 'server_number_output_tokens' NUM_INPUT_TOKENS_SERVER = 'server_number_input_tokens' NUM_TOTAL_TOKENS_SERVER = 'server_number_total_tokens' +NUM_REASONING_TOKENS_SERVER = 'server_number_reasoning_tokens' +NUM_CACHED_TOKENS_SERVER = 'server_number_cached_tokens' diff --git a/benchmarking/src/llmperf/sambanova_client.py b/benchmarking/src/llmperf/sambanova_client.py index 14b14d584..e9d205925 100644 --- a/benchmarking/src/llmperf/sambanova_client.py +++ b/benchmarking/src/llmperf/sambanova_client.py @@ -210,6 +210,13 @@ def _populate_server_metrics(self, response_dict: Dict[str, Any], metrics: Dict[ metrics[common_metrics.NUM_TOTAL_TOKENS_SERVER] = response_dict.get('total_tokens_count') or response_dict.get( 'total_tokens' ) + + completion_tokens_details = response_dict.get('completion_tokens_details') or {} + metrics[common_metrics.NUM_REASONING_TOKENS_SERVER] = completion_tokens_details.get('reasoning_tokens') + + prompt_tokens_details = response_dict.get('prompt_tokens_details') or {} + metrics[common_metrics.NUM_CACHED_TOKENS_SERVER] = prompt_tokens_details.get('cached_tokens') + ttft_server = response_dict.get('time_to_first_token') or response_dict.get('time_to_first_response') metrics[common_metrics.TTFT_SERVER] = ttft_server diff --git a/benchmarking/src/performance_evaluation.py b/benchmarking/src/performance_evaluation.py index 4fbecbf17..6de9365c1 100644 --- a/benchmarking/src/performance_evaluation.py +++ b/benchmarking/src/performance_evaluation.py @@ -291,6 +291,8 @@ def build_metrics_summary( common_metrics.REQ_OUTPUT_THROUGHPUT_SERVER_FIRST_TEN, common_metrics.NUM_INPUT_TOKENS_SERVER, common_metrics.NUM_OUTPUT_TOKENS_SERVER, + common_metrics.NUM_REASONING_TOKENS_SERVER, + common_metrics.NUM_CACHED_TOKENS_SERVER, common_metrics.ACCEPTANCE_RATE, ]: if self.show_results_in_terminal: diff --git a/benchmarking/src/vllm_benchmark.py b/benchmarking/src/vllm_benchmark.py index aad898825..a6e503413 100644 --- a/benchmarking/src/vllm_benchmark.py +++ b/benchmarking/src/vllm_benchmark.py @@ -422,6 +422,8 @@ def _get_error(i: int) -> Optional[str]: 'server_output_token_per_s_per_request': None, 'server_number_output_tokens': None, 'server_number_input_tokens': None, + 'server_number_reasoning_tokens': None, + 'server_number_cached_tokens': None, 'batch_size_used': None, 'error_code': error_str, 'error_msg': error_str, @@ -461,6 +463,8 @@ def _get_error(i: int) -> Optional[str]: 'server_output_token_per_s_per_request': None, 'server_number_output_tokens': None, 'server_number_input_tokens': None, + 'server_number_reasoning_tokens': None, + 'server_number_cached_tokens': None, 'batch_size_used': None, 'error_code': None, 'error_msg': None,