diff --git a/benchmarking/src/llmperf/common_metrics.py b/benchmarking/src/llmperf/common_metrics.py index b679ea200..39ba2ccae 100644 --- a/benchmarking/src/llmperf/common_metrics.py +++ b/benchmarking/src/llmperf/common_metrics.py @@ -29,6 +29,10 @@ MEAN_INTER_TOKEN_LATENCY = 'client_mean_inter_token_latency_s' # Mean ITL per request MEAN_OUTPUT_THROUGHPUT = 'mean_output_throughput_token_per_s' # Mean aggregate throughput +# Client network latency metrics (client-observed minus server-reported timings) +NETWORK_LATENCY_TTFT = 'client_network_latency_ttft_s' # client_ttft_s - server_ttft_s +NETWORK_LATENCY_E2E = 'client_network_latency_e2e_s' # client_end_to_end_latency_s - server_end_to_end_latency_s + # Server-side metrics TTFT_SERVER = 'server_ttft_s' E2E_LAT_SERVER = 'server_end_to_end_latency_s' diff --git a/benchmarking/src/llmperf/sambanova_client.py b/benchmarking/src/llmperf/sambanova_client.py index e9d205925..fdaa3cedb 100644 --- a/benchmarking/src/llmperf/sambanova_client.py +++ b/benchmarking/src/llmperf/sambanova_client.py @@ -160,6 +160,19 @@ def _populate_client_metrics( metrics[common_metrics.E2E_LAT] = total_request_time + # Client network latency: client-observed time minus server-reported time. + # Captures round-trip, request upload, SSE transport, and server queueing. + # Left as None when the server didn't report the corresponding timing. + ttft_server = metrics.get(common_metrics.TTFT_SERVER) + metrics[common_metrics.NETWORK_LATENCY_TTFT] = ( + ttft - ttft_server if ttft_server is not None else None + ) + + e2e_lat_server = metrics.get(common_metrics.E2E_LAT_SERVER) + metrics[common_metrics.NETWORK_LATENCY_E2E] = ( + total_request_time - e2e_lat_server if e2e_lat_server is not None else None + ) + if number_chunks_received == 1: metrics[common_metrics.REQ_OUTPUT_THROUGHPUT] = ( metrics[common_metrics.NUM_OUTPUT_TOKENS] / total_request_time diff --git a/benchmarking/src/performance_evaluation.py b/benchmarking/src/performance_evaluation.py index 6de9365c1..92b3ed896 100644 --- a/benchmarking/src/performance_evaluation.py +++ b/benchmarking/src/performance_evaluation.py @@ -244,11 +244,19 @@ def build_metrics_summary( common_metrics.NUM_INPUT_TOKENS, common_metrics.NUM_OUTPUT_TOKENS, common_metrics.MEAN_INTER_TOKEN_LATENCY, + common_metrics.NETWORK_LATENCY_TTFT, + common_metrics.NETWORK_LATENCY_E2E, ]: if self.show_results_in_terminal: logger.info(f'Building Client Metrics Summary for metric: {metric}') metrics_summary[metric] = {} + # Skip metric if column is absent (for backward compatibility with old result files) + if metric not in metrics_df.columns: + if self.show_results_in_terminal: + logger.info(f' Column {metric} not present, skipping') + continue + # Get flattened list from metric column in metrics df series = pd.Series(list(flatten(metrics_df[metric]))).dropna() @@ -299,6 +307,12 @@ def build_metrics_summary( logger.info(f'Building Server Metrics Summary for metric: {metric}') metrics_summary[metric] = {} + # Skip metric if column is absent (for backward compatibility with old result files) + if metric not in metrics_df.columns: + if self.show_results_in_terminal: + logger.info(f' Column {metric} not present, skipping') + continue + # Get flattened list from metric column in metrics df series = pd.Series(list(flatten(metrics_df[metric]))).dropna()